2024-06-21T00:28:04.370 INFO:root:teuthology version: 0.0.1.dev325+g8e97141 2024-06-21T00:28:04.371 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834 --owner scheduled_lflores@folio02 --name lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --verbose -- /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/orig.config.yaml 2024-06-21T00:28:04.417 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-21T00:28:04.502 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834 branch: squid-release description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: null first_in_suite: false job_id: '7765834' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid-release ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd workunit: branch: squid-release sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd owner: scheduled_lflores@folio02 priority: 75 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5830 sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd sleep_before_teardown: 0 subset: 111/120000 suite: orch suite_branch: squid-release suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd targets: smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5rBkPExFU930dYjxO9KERUYlV86pUbmt/NUHMZD2ttGPI6c7t9NsgJevzFuZWDdqqCd1E5YN6f1EztlpOiI8g= smithi120.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPFx7j5kIN/urguUBHgO++0aUmXqHTL2/oMFPmnm96DMx+054iAXx+utgNidak1lzTYsjyCvh+Gr24MwreuTzSg= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 8e9714173de9e92c97e8ef1045d333e96b793454 timestamp: 2024-06-20_17:49:38 tube: smithi user: lflores verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 2024-06-21T00:28:04.502 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd/qa; will attempt to use it 2024-06-21T00:28:04.503 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd/qa/tasks 2024-06-21T00:28:04.503 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-06-21T00:28:04.505 INFO:teuthology.task.internal:Checking packages... 2024-06-21T00:28:04.526 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd' 2024-06-21T00:28:04.526 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-06-21T00:28:04.527 INFO:teuthology.packaging:ref: None 2024-06-21T00:28:04.527 INFO:teuthology.packaging:tag: None 2024-06-21T00:28:04.527 INFO:teuthology.packaging:branch: squid-release 2024-06-21T00:28:04.527 INFO:teuthology.packaging:sha1: c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T00:28:04.527 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid-release 2024-06-21T00:28:04.752 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2541.gc65be810 2024-06-21T00:28:04.754 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-06-21T00:28:04.763 INFO:teuthology.task.internal:no buildpackages task found 2024-06-21T00:28:04.763 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-06-21T00:28:04.816 INFO:teuthology.task.internal:Saving configuration 2024-06-21T00:28:04.835 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-06-21T00:28:04.860 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-06-21T00:28:04.885 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-21 00:22:18.168300', 'locked_by': 'scheduled_lflores@folio02', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5rBkPExFU930dYjxO9KERUYlV86pUbmt/NUHMZD2ttGPI6c7t9NsgJevzFuZWDdqqCd1E5YN6f1EztlpOiI8g='} 2024-06-21T00:28:04.906 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi120.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-21 00:22:18.166954', 'locked_by': 'scheduled_lflores@folio02', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPFx7j5kIN/urguUBHgO++0aUmXqHTL2/oMFPmnm96DMx+054iAXx+utgNidak1lzTYsjyCvh+Gr24MwreuTzSg='} 2024-06-21T00:28:04.906 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-06-21T00:28:04.915 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-06-21T00:28:04.915 INFO:teuthology.task.internal:roles: ubuntu@smithi120.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-06-21T00:28:04.915 INFO:teuthology.run_tasks:Running task console_log... 2024-06-21T00:28:05.002 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fac72f57ac0>, signals=[15]) 2024-06-21T00:28:05.002 INFO:teuthology.run_tasks:Running task internal.connect... 2024-06-21T00:28:05.042 INFO:teuthology.task.internal:Opening connections... 2024-06-21T00:28:05.042 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2024-06-21T00:28:05.045 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:28:05.127 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi120.front.sepia.ceph.com 2024-06-21T00:28:05.127 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:28:05.206 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-06-21T00:28:05.216 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2024-06-21T00:28:05.238 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2024-06-21T00:28:05.238 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2024-06-21T00:28:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="CentOS Stream" 2024-06-21T00:28:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="9" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:ID="centos" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="rhel fedora" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="9" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el9" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-21T00:28:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2024-06-21T00:28:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:LOGO="fedora-logo-icon" 2024-06-21T00:28:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-21T00:28:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://centos.org/" 2024-06-21T00:28:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-21T00:28:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-21T00:28:05.299 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-21T00:28:05.300 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2024-06-21T00:28:05.324 DEBUG:teuthology.orchestra.run.smithi120:> uname -m 2024-06-21T00:28:05.343 INFO:teuthology.orchestra.run.smithi120.stdout:x86_64 2024-06-21T00:28:05.344 DEBUG:teuthology.orchestra.run.smithi120:> cat /etc/os-release 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:NAME="CentOS Stream" 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:VERSION="9" 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:ID="centos" 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:ID_LIKE="rhel fedora" 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:VERSION_ID="9" 2024-06-21T00:28:05.400 INFO:teuthology.orchestra.run.smithi120.stdout:PLATFORM_ID="platform:el9" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:ANSI_COLOR="0;31" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:LOGO="fedora-logo-icon" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:HOME_URL="https://centos.org/" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-21T00:28:05.401 INFO:teuthology.orchestra.run.smithi120.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-21T00:28:05.402 INFO:teuthology.lock.ops:Updating smithi120.front.sepia.ceph.com on lock server 2024-06-21T00:28:05.424 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-06-21T00:28:05.437 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-06-21T00:28:05.508 INFO:teuthology.task.internal:Checking for old test directory... 2024-06-21T00:28:05.508 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2024-06-21T00:28:05.511 DEBUG:teuthology.orchestra.run.smithi120:> test '!' -e /home/ubuntu/cephtest 2024-06-21T00:28:05.527 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-06-21T00:28:05.560 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-06-21T00:28:05.561 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2024-06-21T00:28:05.568 DEBUG:teuthology.orchestra.run.smithi120:> test -z $(ls -A /var/lib/ceph) 2024-06-21T00:28:05.591 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-06-21T00:28:05.686 INFO:teuthology.run_tasks:Running task kernel... 2024-06-21T00:28:05.705 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-06-21T00:28:05.705 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-06-21T00:28:05.706 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-21T00:28:05.706 DEBUG:teuthology.orchestra.run.smithi120:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-21T00:28:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T00:28:05.723 DEBUG:teuthology.orchestra.run.smithi120:> uname -r 2024-06-21T00:28:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T00:28:05.725 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-06-21T00:28:05.781 INFO:teuthology.orchestra.run.smithi120.stdout:5.14.0-437.el9.x86_64 2024-06-21T00:28:05.781 INFO:teuthology.task.kernel:Running kernel on smithi120: 5.14.0-437.el9.x86_64 2024-06-21T00:28:05.782 DEBUG:teuthology.orchestra.run.smithi120:> sudo yum install -y kernel 2024-06-21T00:28:05.783 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-437.el9.x86_64 2024-06-21T00:28:05.783 INFO:teuthology.task.kernel:Running kernel on smithi027: 5.14.0-437.el9.x86_64 2024-06-21T00:28:05.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2024-06-21T00:28:08.153 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.1 MB 00:00 2024-06-21T00:28:09.879 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - AppStream 35 MB/s | 20 MB 00:00 2024-06-21T00:28:10.803 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 9 - BaseOS 2.4 MB/s | 8.1 MB 00:03 2024-06-21T00:28:15.638 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 9 - AppStream 5.3 MB/s | 20 MB 00:03 2024-06-21T00:28:15.655 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - CRB 3.3 MB/s | 6.3 MB 00:01 2024-06-21T00:28:17.564 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - Extras packages 25 kB/s | 17 kB 00:00 2024-06-21T00:28:18.426 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 30 MB/s | 21 MB 00:00 2024-06-21T00:28:21.255 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 9 - CRB 3.4 MB/s | 6.3 MB 00:01 2024-06-21T00:28:23.055 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 9 - Extras packages 36 kB/s | 17 kB 00:00 2024-06-21T00:28:23.807 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-06-21T00:28:23.943 INFO:teuthology.orchestra.run.smithi120.stdout:Extra Packages for Enterprise Linux 29 MB/s | 21 MB 00:00 2024-06-21T00:28:25.972 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-21T00:28:25.972 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-21T00:28:26.077 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 5.14.0-460.el9 baseos 1.1 M 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core x86_64 5.14.0-460.el9 baseos 16 M 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules x86_64 5.14.0-460.el9 baseos 35 M 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core x86_64 5.14.0-460.el9 baseos 29 M 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-21T00:28:26.085 INFO:teuthology.orchestra.run.smithi027.stdout:Install 4 Packages 2024-06-21T00:28:26.086 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:28:26.086 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 81 M 2024-06-21T00:28:26.086 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 124 M 2024-06-21T00:28:26.086 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-06-21T00:28:26.742 INFO:teuthology.orchestra.run.smithi027.stdout:(1/4): kernel-5.14.0-460.el9.x86_64.rpm 2.8 MB/s | 1.1 MB 00:00 2024-06-21T00:28:29.157 INFO:teuthology.orchestra.run.smithi120.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-06-21T00:28:31.205 INFO:teuthology.orchestra.run.smithi120.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-21T00:28:31.206 INFO:teuthology.orchestra.run.smithi120.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-21T00:28:31.293 INFO:teuthology.orchestra.run.smithi120.stdout:Dependencies resolved. 2024-06-21T00:28:31.305 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-06-21T00:28:31.305 INFO:teuthology.orchestra.run.smithi120.stdout: Package Architecture Version Repository Size 2024-06-21T00:28:31.305 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-06-21T00:28:31.305 INFO:teuthology.orchestra.run.smithi120.stdout:Installing: 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: kernel x86_64 5.14.0-460.el9 baseos 1.1 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Installing dependencies: 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-core x86_64 5.14.0-460.el9 baseos 16 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules x86_64 5.14.0-460.el9 baseos 35 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules-core x86_64 5.14.0-460.el9 baseos 29 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction Summary 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Install 4 Packages 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Total download size: 81 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Installed size: 124 M 2024-06-21T00:28:31.306 INFO:teuthology.orchestra.run.smithi120.stdout:Downloading Packages: 2024-06-21T00:28:31.767 INFO:teuthology.orchestra.run.smithi120.stdout:(1/4): kernel-5.14.0-460.el9.x86_64.rpm 3.1 MB/s | 1.1 MB 00:00 2024-06-21T00:28:34.141 INFO:teuthology.orchestra.run.smithi027.stdout:(2/4): kernel-core-5.14.0-460.el9.x86_64.rpm 2.1 MB/s | 16 MB 00:07 2024-06-21T00:28:35.517 INFO:teuthology.orchestra.run.smithi027.stdout:(3/4): kernel-modules-core-5.14.0-460.el9.x86_6 3.3 MB/s | 29 MB 00:08 2024-06-21T00:28:39.183 INFO:teuthology.orchestra.run.smithi120.stdout:(2/4): kernel-core-5.14.0-460.el9.x86_64.rpm 2.1 MB/s | 16 MB 00:07 2024-06-21T00:28:45.183 INFO:teuthology.orchestra.run.smithi027.stdout:(4/4): kernel-modules-5.14.0-460.el9.x86_64.rpm 1.8 MB/s | 35 MB 00:18 2024-06-21T00:28:45.183 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-06-21T00:28:45.183 INFO:teuthology.orchestra.run.smithi027.stdout:Total 4.2 MB/s | 81 MB 00:19 2024-06-21T00:28:45.922 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-06-21T00:28:46.345 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-06-21T00:28:46.345 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-06-21T00:28:47.343 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-06-21T00:28:47.344 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-06-21T00:28:47.383 INFO:teuthology.orchestra.run.smithi120.stdout:(3/4): kernel-modules-core-5.14.0-460.el9.x86_6 1.9 MB/s | 29 MB 00:15 2024-06-21T00:28:49.574 INFO:teuthology.orchestra.run.smithi120.stdout:(4/4): kernel-modules-5.14.0-460.el9.x86_64.rpm 1.9 MB/s | 35 MB 00:18 2024-06-21T00:28:49.575 INFO:teuthology.orchestra.run.smithi120.stdout:-------------------------------------------------------------------------------- 2024-06-21T00:28:49.575 INFO:teuthology.orchestra.run.smithi120.stdout:Total 4.4 MB/s | 81 MB 00:18 2024-06-21T00:28:49.918 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-06-21T00:28:50.280 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction check 2024-06-21T00:28:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-core-5.14.0-460.el9.x86_64 1/4 2024-06-21T00:28:50.620 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction check succeeded. 2024-06-21T00:28:50.620 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction test 2024-06-21T00:28:50.666 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:28:51.586 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction test succeeded. 2024-06-21T00:28:51.587 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction 2024-06-21T00:28:53.457 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:28:53.612 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:28:53.989 INFO:teuthology.orchestra.run.smithi120.stdout: Preparing : 1/1 2024-06-21T00:28:54.641 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-modules-core-5.14.0-460.el9.x86_64 1/4 2024-06-21T00:28:54.736 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:28:57.377 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:28:57.531 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:28:58.717 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:28:58.778 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:02.266 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:29:02.403 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:03.913 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:07.149 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:35.153 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:35.153 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-5.14.0-460.el9.x86_64 1/4 2024-06-21T00:29:35.154 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:29:35.154 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:29:35.981 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:35.981 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:29:35.981 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core-5.14.0-460.el9.x86_64 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-5.14.0-460.el9.x86_64 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core-5.14.0-460.el9.x86_64 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:29:35.982 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-06-21T00:29:36.288 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:36.329 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-modules-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:36.472 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-06-21T00:29:36.572 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:36.573 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-21T00:29:36.573 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-460.el9.x86_64 2024-06-21T00:29:36.573 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-21T00:29:36.573 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-21T00:29:36.573 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-06-21T00:29:36.574 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi027.front.sepia.ceph.com, path=None, version=distro) 2024-06-21T00:29:36.574 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-06-21T00:29:36.628 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:36.629 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-21T00:29:36.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo rpm -qi grub2-tools 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Name : grub2-tools 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Epoch : 1 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Version : 2.06 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Release : 68.el9 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Architecture: x86_64 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Group : Unspecified 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:Size : 8264254 2024-06-21T00:29:36.661 INFO:teuthology.orchestra.run.smithi027.stdout:License : GPLv3+ 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Packager : builder@centos.org 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Vendor : CentOS 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Summary : Support tools for GRUB. 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:Description : 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-21T00:29:36.662 INFO:teuthology.orchestra.run.smithi027.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-21T00:29:36.663 INFO:teuthology.orchestra.run.smithi027.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-21T00:29:36.663 INFO:teuthology.orchestra.run.smithi027.stdout:hardware devices. 2024-06-21T00:29:36.663 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:29:36.663 INFO:teuthology.orchestra.run.smithi027.stdout:This subpackage provides tools for support of all platforms. 2024-06-21T00:29:36.664 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-21T00:29:36.664 INFO:teuthology.task.kernel:Updating grub on smithi027 to boot 5.14.0-460.el9.x86_64 2024-06-21T00:29:36.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-21T00:29:36.686 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:36.686 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-5.14.0-460.el9.x86_64 1/4 2024-06-21T00:29:36.686 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-core-5.14.0-460.el9.x86_64 2/4 2024-06-21T00:29:36.686 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-modules-5.14.0-460.el9.x86_64 3/4 2024-06-21T00:29:37.283 INFO:teuthology.orchestra.run.smithi027.stderr:Generating grub configuration file ... 2024-06-21T00:29:37.458 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-modules-core-5.14.0-460.el9.x86_64 4/4 2024-06-21T00:29:37.458 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:29:37.458 INFO:teuthology.orchestra.run.smithi120.stdout:Installed: 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-core-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules-core-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:29:37.459 INFO:teuthology.orchestra.run.smithi120.stdout:Complete! 2024-06-21T00:29:37.844 DEBUG:teuthology.orchestra.run.smithi120:> rpm -q kernel | sort -rV | head -n 1 2024-06-21T00:29:37.922 INFO:teuthology.orchestra.run.smithi120.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.922 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-21T00:29:37.922 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-460.el9.x86_64 2024-06-21T00:29:37.923 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-21T00:29:37.923 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-21T00:29:37.923 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-06-21T00:29:37.923 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi120.front.sepia.ceph.com, path=None, version=distro) 2024-06-21T00:29:37.923 DEBUG:teuthology.orchestra.run.smithi120:> rpm -q kernel | sort -rV | head -n 1 2024-06-21T00:29:37.976 INFO:teuthology.orchestra.run.smithi120.stdout:kernel-5.14.0-460.el9.x86_64 2024-06-21T00:29:37.976 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-460.el9.x86_64 2024-06-21T00:29:37.976 DEBUG:teuthology.orchestra.run.smithi120:> sudo rpm -qi grub2-tools 2024-06-21T00:29:38.007 INFO:teuthology.orchestra.run.smithi120.stdout:Name : grub2-tools 2024-06-21T00:29:38.007 INFO:teuthology.orchestra.run.smithi120.stdout:Epoch : 1 2024-06-21T00:29:38.007 INFO:teuthology.orchestra.run.smithi120.stdout:Version : 2.06 2024-06-21T00:29:38.007 INFO:teuthology.orchestra.run.smithi120.stdout:Release : 68.el9 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Architecture: x86_64 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Group : Unspecified 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Size : 8264254 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:License : GPLv3+ 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Packager : builder@centos.org 2024-06-21T00:29:38.008 INFO:teuthology.orchestra.run.smithi120.stdout:Vendor : CentOS 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:Summary : Support tools for GRUB. 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:Description : 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:hardware devices. 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:29:38.009 INFO:teuthology.orchestra.run.smithi120.stdout:This subpackage provides tools for support of all platforms. 2024-06-21T00:29:38.010 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-21T00:29:38.010 INFO:teuthology.task.kernel:Updating grub on smithi120 to boot 5.14.0-460.el9.x86_64 2024-06-21T00:29:38.010 DEBUG:teuthology.orchestra.run.smithi120:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-21T00:29:38.577 INFO:teuthology.orchestra.run.smithi120.stderr:Generating grub configuration file ... 2024-06-21T00:29:38.622 INFO:teuthology.orchestra.run.smithi027.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-21T00:29:38.651 INFO:teuthology.orchestra.run.smithi027.stderr:done 2024-06-21T00:29:38.654 DEBUG:teuthology.orchestra.run.smithi027:> mktemp 2024-06-21T00:29:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:/tmp/tmp.EFAEwvY1cz 2024-06-21T00:29:38.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.EFAEwvY1cz 2024-06-21T00:29:38.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0666 /tmp/tmp.EFAEwvY1cz 2024-06-21T00:29:38.925 DEBUG:teuthology.orchestra.remote:smithi027:/tmp/tmp.EFAEwvY1cz is 6KB 2024-06-21T00:29:38.973 DEBUG:teuthology.orchestra.run.smithi027:> rm -fr /tmp/tmp.EFAEwvY1cz 2024-06-21T00:29:38.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo /bin/ls /boot/loader/entries || true 2024-06-21T00:29:39.050 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-21T00:29:39.050 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-21T00:29:39.051 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-21T00:29:39.051 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64.conf 2024-06-21T00:29:39.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64 2024-06-21T00:29:39.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo shutdown -r now 2024-06-21T00:29:40.008 INFO:teuthology.orchestra.run.smithi120.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-21T00:29:40.032 INFO:teuthology.orchestra.run.smithi120.stderr:done 2024-06-21T00:29:40.037 DEBUG:teuthology.orchestra.run.smithi120:> mktemp 2024-06-21T00:29:40.053 INFO:teuthology.orchestra.run.smithi120.stdout:/tmp/tmp.xC94CyawJF 2024-06-21T00:29:40.054 DEBUG:teuthology.orchestra.run.smithi120:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.xC94CyawJF 2024-06-21T00:29:40.118 DEBUG:teuthology.orchestra.run.smithi120:> sudo chmod 0666 /tmp/tmp.xC94CyawJF 2024-06-21T00:29:40.252 DEBUG:teuthology.orchestra.remote:smithi120:/tmp/tmp.xC94CyawJF is 6KB 2024-06-21T00:29:40.271 DEBUG:teuthology.orchestra.run.smithi120:> rm -fr /tmp/tmp.xC94CyawJF 2024-06-21T00:29:40.285 DEBUG:teuthology.orchestra.run.smithi120:> sudo /bin/ls /boot/loader/entries || true 2024-06-21T00:29:40.348 INFO:teuthology.orchestra.run.smithi120.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-21T00:29:40.348 INFO:teuthology.orchestra.run.smithi120.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-21T00:29:40.348 INFO:teuthology.orchestra.run.smithi120.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-21T00:29:40.349 INFO:teuthology.orchestra.run.smithi120.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64.conf 2024-06-21T00:29:40.350 DEBUG:teuthology.orchestra.run.smithi120:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-460.el9.x86_64 2024-06-21T00:29:40.500 DEBUG:teuthology.orchestra.run.smithi120:> sudo shutdown -r now 2024-06-21T00:30:09.226 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-06-21T00:30:09.227 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:30:09.228 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:10.503 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-06-21T00:30:10.504 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:30:10.504 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:27.621 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-21T00:30:28.894 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.120 2024-06-21T00:30:36.630 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:30:36.631 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:37.896 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:30:37.897 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:39.713 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-21T00:30:40.966 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.120 2024-06-21T00:30:51.717 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:30:51.719 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:52.968 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:30:52.968 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:30:54.782 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-21T00:31:09.792 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:31:09.793 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:31:11.390 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.120 2024-06-21T00:31:26.405 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:31:26.406 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:31:26.805 DEBUG:teuthology.orchestra.run.smithi120:> true 2024-06-21T00:31:27.157 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:31:27.157 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-460.el9.x86_64"... 2024-06-21T00:31:27.157 DEBUG:teuthology.orchestra.run.smithi120:> uname -r 2024-06-21T00:31:27.172 INFO:teuthology.orchestra.run.smithi120.stdout:5.14.0-460.el9.x86_64 2024-06-21T00:31:27.173 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-460.el9.x86_64 vs 5.14.0-460.el9.x86_64 2024-06-21T00:31:27.173 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-21T00:31:27.173 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-21T00:31:28.173 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-06-21T00:31:28.174 DEBUG:teuthology.orchestra.run.smithi120:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-21T00:31:28.258 INFO:teuthology.orchestra.run.smithi120.stdout:ttyS1 2024-06-21T00:31:28.286 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-21T00:31:28.293 DEBUG:teuthology.parallel:result is None 2024-06-21T00:31:46.288 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:31:46.289 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:31:46.625 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-06-21T00:31:47.174 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:31:47.174 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-460.el9.x86_64"... 2024-06-21T00:31:47.174 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-06-21T00:31:47.201 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-460.el9.x86_64 2024-06-21T00:31:47.202 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-460.el9.x86_64 vs 5.14.0-460.el9.x86_64 2024-06-21T00:31:47.202 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-21T00:31:47.202 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-21T00:31:48.203 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-06-21T00:31:48.203 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-21T00:31:48.262 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2024-06-21T00:31:48.306 DEBUG:teuthology.parallel:result is None 2024-06-21T00:31:48.306 INFO:teuthology.run_tasks:Running task internal.base... 2024-06-21T00:31:48.327 INFO:teuthology.task.internal:Creating test directory... 2024-06-21T00:31:48.328 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-21T00:31:48.330 DEBUG:teuthology.orchestra.run.smithi120:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-21T00:31:48.350 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-06-21T00:31:48.368 INFO:teuthology.run_tasks:Running task internal.archive... 2024-06-21T00:31:48.394 INFO:teuthology.task.internal:Creating archive directory... 2024-06-21T00:31:48.395 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-21T00:31:48.397 DEBUG:teuthology.orchestra.run.smithi120:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-21T00:31:48.512 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-06-21T00:31:48.521 INFO:teuthology.task.internal:Enabling coredump saving... 2024-06-21T00:31:48.522 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-21T00:31:48.524 DEBUG:teuthology.orchestra.run.smithi120:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-21T00:31:48.559 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-21T00:31:48.571 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-21T00:31:48.651 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-21T00:31:48.662 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-21T00:31:48.663 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-06-21T00:31:48.691 INFO:teuthology.task.internal:Configuring sudo... 2024-06-21T00:31:48.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-21T00:31:48.693 DEBUG:teuthology.orchestra.run.smithi120:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-21T00:31:48.723 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-06-21T00:31:48.749 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-06-21T00:31:48.750 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-21T00:31:48.759 DEBUG:teuthology.orchestra.run.smithi120:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-21T00:31:48.781 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-21T00:31:48.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-21T00:31:48.956 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-21T00:31:49.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-21T00:31:49.040 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:31:49.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-21T00:31:49.104 DEBUG:teuthology.orchestra.run.smithi120:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-21T00:31:49.148 DEBUG:teuthology.orchestra.run.smithi120:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-21T00:31:49.202 DEBUG:teuthology.orchestra.run.smithi120:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-21T00:31:49.287 DEBUG:teuthology.orchestra.run.smithi120:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-21T00:31:49.331 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:31:49.331 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-21T00:31:49.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2024-06-21T00:31:49.401 DEBUG:teuthology.orchestra.run.smithi120:> sudo service rsyslog restart 2024-06-21T00:31:49.446 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-21T00:31:49.480 INFO:teuthology.orchestra.run.smithi120.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-21T00:31:49.906 INFO:teuthology.run_tasks:Running task internal.timer... 2024-06-21T00:31:49.915 INFO:teuthology.task.internal:Starting timer... 2024-06-21T00:31:49.916 INFO:teuthology.run_tasks:Running task pcp... 2024-06-21T00:31:49.970 INFO:teuthology.run_tasks:Running task selinux... 2024-06-21T00:31:50.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2024-06-21T00:31:50.070 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2024-06-21T00:31:50.072 DEBUG:teuthology.orchestra.run.smithi120:> sudo service auditd rotate 2024-06-21T00:31:50.136 INFO:teuthology.orchestra.run.smithi120.stdout:Rotating logs: 2024-06-21T00:31:50.138 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-06-21T00:31:50.139 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2024-06-21T00:31:50.165 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2024-06-21T00:31:50.165 DEBUG:teuthology.orchestra.run.smithi120:> /usr/sbin/getenforce 2024-06-21T00:31:50.197 INFO:teuthology.orchestra.run.smithi120.stdout:Permissive 2024-06-21T00:31:50.198 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi027.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi120.front.sepia.ceph.com': 'permissive'} 2024-06-21T00:31:50.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-21T00:31:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T00:31:50.232 DEBUG:teuthology.orchestra.run.smithi120:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-21T00:31:50.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T00:31:50.266 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-06-21T00:31:50.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2024-06-21T00:31:50.299 DEBUG:teuthology.orchestra.run.smithi120:> sudo /usr/sbin/setenforce permissive 2024-06-21T00:31:50.341 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-06-21T00:31:50.350 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-06-21T00:31:50.399 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-06-21T00:31:50.399 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi027.front.sepia.ceph.com,smithi120.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-06-21T00:36:19.590 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi027.front.sepia.ceph.com'), Remote(name='ubuntu@smithi120.front.sepia.ceph.com')] 2024-06-21T00:36:19.620 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:36:19.621 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:36:19.698 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-06-21T00:36:19.779 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-21T00:36:19.779 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:36:19.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-21T00:36:19.856 DEBUG:teuthology.orchestra.run.smithi120:> true 2024-06-21T00:36:19.933 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-06-21T00:36:19.934 INFO:teuthology.run_tasks:Running task clock... 2024-06-21T00:36:19.945 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-06-21T00:36:19.946 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-21T00:36:19.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-21T00:36:19.948 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-21T00:36:19.949 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-21T00:36:19.980 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-21T00:36:19.996 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-21T00:36:20.004 INFO:teuthology.orchestra.run.smithi120.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-21T00:36:20.018 INFO:teuthology.orchestra.run.smithi120.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-21T00:36:20.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: ntpd: command not found 2024-06-21T00:36:20.035 INFO:teuthology.orchestra.run.smithi027.stdout:506 Cannot talk to daemon 2024-06-21T00:36:20.042 INFO:teuthology.orchestra.run.smithi120.stderr:sudo: ntpd: command not found 2024-06-21T00:36:20.050 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-21T00:36:20.053 INFO:teuthology.orchestra.run.smithi120.stdout:506 Cannot talk to daemon 2024-06-21T00:36:20.065 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-21T00:36:20.067 INFO:teuthology.orchestra.run.smithi120.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-21T00:36:20.081 INFO:teuthology.orchestra.run.smithi120.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-21T00:36:20.129 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-06-21T00:36:20.133 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-21T00:36:20.133 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-06-21T00:36:20.133 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.133 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.133 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.134 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.138 INFO:teuthology.orchestra.run.smithi120.stderr:bash: line 1: ntpq: command not found 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:=============================================================================== 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.142 INFO:teuthology.orchestra.run.smithi120.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T00:36:20.143 INFO:teuthology.run_tasks:Running task cephadm... 2024-06-21T00:36:20.274 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd'} 2024-06-21T00:36:20.274 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-06-21T00:36:20.275 INFO:tasks.cephadm:Cluster fsid is 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:20.275 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-06-21T00:36:20.275 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.27', 'mon.c': '[v2:172.21.15.27:3301,v1:172.21.15.27:6790]', 'mon.b': '172.21.15.120'} 2024-06-21T00:36:20.275 INFO:tasks.cephadm:First mon is mon.a on smithi027 2024-06-21T00:36:20.275 INFO:tasks.cephadm:First mgr is y 2024-06-21T00:36:20.275 INFO:tasks.cephadm:Normalizing hostnames... 2024-06-21T00:36:20.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo hostname $(hostname -s) 2024-06-21T00:36:20.303 DEBUG:teuthology.orchestra.run.smithi120:> sudo hostname $(hostname -s) 2024-06-21T00:36:20.329 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-06-21T00:36:20.329 DEBUG:teuthology.orchestra.run.smithi027:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.627 INFO:teuthology.orchestra.run.smithi027.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 21 00:36 /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.627 DEBUG:teuthology.orchestra.run.smithi120:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.795 INFO:teuthology.orchestra.run.smithi120.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 21 00:36 /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.797 DEBUG:teuthology.orchestra.run.smithi027:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.815 DEBUG:teuthology.orchestra.run.smithi120:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-21T00:36:20.844 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-06-21T00:36:20.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-21T00:36:20.858 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-21T00:36:21.720 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-21T00:36:21.769 INFO:teuthology.orchestra.run.smithi120.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-21T00:36:37.731 INFO:teuthology.orchestra.run.smithi120.stdout:{ 2024-06-21T00:36:37.731 INFO:teuthology.orchestra.run.smithi120.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-21T00:36:37.731 INFO:teuthology.orchestra.run.smithi120.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-21T00:36:37.732 INFO:teuthology.orchestra.run.smithi120.stdout: "repo_digests": [ 2024-06-21T00:36:37.732 INFO:teuthology.orchestra.run.smithi120.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-21T00:36:37.732 INFO:teuthology.orchestra.run.smithi120.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-21T00:36:37.732 INFO:teuthology.orchestra.run.smithi120.stdout: ] 2024-06-21T00:36:37.732 INFO:teuthology.orchestra.run.smithi120.stdout:} 2024-06-21T00:36:41.965 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:36:41.965 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-21T00:36:41.965 INFO:teuthology.orchestra.run.smithi027.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-21T00:36:41.965 INFO:teuthology.orchestra.run.smithi027.stdout: "repo_digests": [ 2024-06-21T00:36:41.966 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-21T00:36:41.966 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-21T00:36:41.966 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-06-21T00:36:41.966 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:36:41.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo mkdir -p /etc/ceph 2024-06-21T00:36:42.028 DEBUG:teuthology.orchestra.run.smithi120:> sudo mkdir -p /etc/ceph 2024-06-21T00:36:42.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 777 /etc/ceph 2024-06-21T00:36:42.093 DEBUG:teuthology.orchestra.run.smithi120:> sudo chmod 777 /etc/ceph 2024-06-21T00:36:42.130 INFO:tasks.cephadm:Writing seed config... 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-06-21T00:36:42.131 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-06-21T00:36:42.132 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-06-21T00:36:42.132 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:36:42.132 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-06-21T00:36:42.149 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 47651940-2f66-11ef-bc9d-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-06-21T00:36:42.150 DEBUG:teuthology.orchestra.run.smithi027:mon.a> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service 2024-06-21T00:36:42.192 DEBUG:teuthology.orchestra.run.smithi027:mgr.y> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service 2024-06-21T00:36:42.234 INFO:tasks.cephadm:Bootstrapping... 2024-06-21T00:36:42.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 47651940-2f66-11ef-bc9d-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.27 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-06-21T00:36:42.435 INFO:teuthology.orchestra.run.smithi027.stderr:-------------------------------------------------------------------------------- 2024-06-21T00:36:42.435 INFO:teuthology.orchestra.run.smithi027.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '47651940-2f66-11ef-bc9d-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.27', '--skip-admin-label'] 2024-06-21T00:36:42.459 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: 5.1.0 2024-06-21T00:36:42.461 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying podman|docker is present... 2024-06-21T00:36:42.484 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: 5.1.0 2024-06-21T00:36:42.485 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying lvm2 is present... 2024-06-21T00:36:42.486 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying time synchronization is in place... 2024-06-21T00:36:42.493 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-21T00:36:42.499 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:36:42.506 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: enabled 2024-06-21T00:36:42.513 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: active 2024-06-21T00:36:42.514 INFO:teuthology.orchestra.run.smithi027.stderr:Unit chronyd.service is enabled and running 2024-06-21T00:36:42.514 INFO:teuthology.orchestra.run.smithi027.stderr:Repeating the final host check... 2024-06-21T00:36:42.535 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: 5.1.0 2024-06-21T00:36:42.537 INFO:teuthology.orchestra.run.smithi027.stderr:podman (/bin/podman) version 5.1.0 is present 2024-06-21T00:36:42.537 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl is present 2024-06-21T00:36:42.537 INFO:teuthology.orchestra.run.smithi027.stderr:lvcreate is present 2024-06-21T00:36:42.543 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-21T00:36:42.550 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:36:42.557 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: enabled 2024-06-21T00:36:42.563 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: active 2024-06-21T00:36:42.564 INFO:teuthology.orchestra.run.smithi027.stderr:Unit chronyd.service is enabled and running 2024-06-21T00:36:42.564 INFO:teuthology.orchestra.run.smithi027.stderr:Host looks OK 2024-06-21T00:36:42.564 INFO:teuthology.orchestra.run.smithi027.stderr:Cluster fsid: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:Acquiring lock 140014085329920 on /run/cephadm/47651940-2f66-11ef-bc9d-c7b262605968.lock 2024-06-21T00:36:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:Lock 140014085329920 acquired on /run/cephadm/47651940-2f66-11ef-bc9d-c7b262605968.lock 2024-06-21T00:36:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying IP 172.21.15.27 port 3300 ... 2024-06-21T00:36:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying IP 172.21.15.27 port 6789 ... 2024-06-21T00:36:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:Base mon IP is 172.21.15.27, final addrv is [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-06-21T00:36:42.567 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.27 metric 100 2024-06-21T00:36:42.568 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.27 metric 100 2024-06-21T00:36:42.570 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-06-21T00:36:42.570 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-06-21T00:36:42.571 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-06-21T00:36:42.573 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-06-21T00:36:42.573 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-06-21T00:36:42.573 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-21T00:36:42.573 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-06-21T00:36:42.573 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:395a/64 scope link noprefixroute 2024-06-21T00:36:42.574 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-21T00:36:42.574 INFO:teuthology.orchestra.run.smithi027.stderr:Mon IP `172.21.15.27` is in CIDR network `172.21.0.0/20` 2024-06-21T00:36:42.574 INFO:teuthology.orchestra.run.smithi027.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-06-21T00:36:42.575 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-21T00:36:42.680 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-06-21T00:36:43.165 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Getting image source signatures 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-06-21T00:36:43.166 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-21T00:36:43.318 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Writing manifest to image destination 2024-06-21T00:36:43.430 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-21T00:36:43.818 INFO:teuthology.orchestra.run.smithi027.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-21T00:36:44.354 INFO:teuthology.orchestra.run.smithi027.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-21T00:36:44.354 INFO:teuthology.orchestra.run.smithi027.stderr:Extracting ceph user uid/gid from container image... 2024-06-21T00:36:44.753 INFO:teuthology.orchestra.run.smithi027.stderr:stat: 167 167 2024-06-21T00:36:45.207 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial keys... 2024-06-21T00:36:45.609 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAdy3RmqBaDHxAATc3KbG1kCkbeNIMz/MIX9Q== 2024-06-21T00:36:46.465 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAey3Rm7MTsFhAACaW+XJq1Q+JhWjpaU1v8dQ== 2024-06-21T00:36:47.329 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAfy3RmunFaDhAAFa5cnSkKkgBGCbV9tzedqw== 2024-06-21T00:36:47.752 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial monmap... 2024-06-21T00:36:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-21T00:36:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-06-21T00:36:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr:monmaptool for a [v2:172.21.15.27:3300,v1:172.21.15.27:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr:setting min_mon_release = octopus 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: set fsid to 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:36:48.709 INFO:teuthology.orchestra.run.smithi027.stderr:Creating mon... 2024-06-21T00:36:49.640 INFO:teuthology.orchestra.run.smithi027.stderr:create mon.a on 2024-06-21T00:36:49.863 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-06-21T00:36:50.017 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target → /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target. 2024-06-21T00:36:50.018 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target → /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target. 2024-06-21T00:36:50.346 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to reset failed state of unit ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service: Unit ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service not loaded. 2024-06-21T00:36:50.354 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target.wants/ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service → /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968@.service. 2024-06-21T00:36:51.079 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-06-21T00:36:51.087 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:36:51.088 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-06-21T00:36:51.088 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to enable service . firewalld.service is not available 2024-06-21T00:36:51.088 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mon to start... 2024-06-21T00:36:51.088 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mon... 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: cluster: 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: id: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: services: 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.535065s) 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr: no daemons active 2024-06-21T00:36:51.569 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: data: 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: pgs: 2024-06-21T00:36:51.570 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:52.067 INFO:teuthology.orchestra.run.smithi027.stderr:mon is available 2024-06-21T00:36:52.067 INFO:teuthology.orchestra.run.smithi027.stderr:Assimilating anything we can from ceph.conf... 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [global] 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: fsid = 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:52.605 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [mgr] 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [osd] 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-21T00:36:52.606 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-21T00:36:53.147 INFO:teuthology.orchestra.run.smithi027.stderr:Generating new minimal ceph.conf... 2024-06-21T00:36:54.121 INFO:teuthology.orchestra.run.smithi027.stderr:Restarting the monitor... 2024-06-21T00:36:54.302 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a[24788]: 2024-06-21T00:36:54.272+0000 7f8277838700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-06-21T00:36:54.302 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 podman[25023]: 2024-06-21 00:36:54.303467796 +0000 UTC m=+0.119854170 container died 0ec0383661cb226a60bb77c30aa5f95a053eb2be93647fdaab09e153cdd007f6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, build-date=2022-05-03T08:36:31.336870) 2024-06-21T00:36:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 podman[25023]: 2024-06-21 00:36:54.78885433 +0000 UTC m=+0.605240696 container cleanup 0ec0383661cb226a60bb77c30aa5f95a053eb2be93647fdaab09e153cdd007f6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8) 2024-06-21T00:36:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 bash[25023]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a 2024-06-21T00:36:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 bash[25051]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon.a" found: no such container 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 podman[25038]: 2024-06-21 00:36:54.989602359 +0000 UTC m=+0.689890144 container remove 0ec0383661cb226a60bb77c30aa5f95a053eb2be93647fdaab09e153cdd007f6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service: Deactivated successfully. 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:54 smithi027 systemd[1]: Stopped Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 systemd[1]: Starting Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 podman[25117]: 2024-06-21 00:36:55.135594355 +0000 UTC m=+0.018629056 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-06-21T00:36:55.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 podman[25117]: 2024-06-21 00:36:55.232333333 +0000 UTC m=+0.115368057 container create 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, vcs-type=git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, GIT_CLEAN=True) 2024-06-21T00:36:55.527 INFO:teuthology.orchestra.run.smithi027.stderr:Setting mon public_network to 172.21.0.0/20 2024-06-21T00:36:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 podman[25117]: 2024-06-21 00:36:55.45133824 +0000 UTC m=+0.334372940 container init 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, ceph=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-21T00:36:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 podman[25117]: 2024-06-21 00:36:55.454538656 +0000 UTC m=+0.337573344 container start 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, RELEASE=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, name=centos-stream, maintainer=Guillaume Abrioux ) 2024-06-21T00:36:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: set uid:gid to 167:167 (ceph:ceph) 2024-06-21T00:36:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-21T00:36:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: pidfile_write: ignore empty --pid-file 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: load: jerasure load: lrc 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: RocksDB version: 6.15.5 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Compile date Apr 18 2022 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: DB SUMMARY 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: DB Session ID: HIZ5JZSX3P56CY849KUF 2024-06-21T00:36:55.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: CURRENT file: CURRENT 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: IDENTITY file: IDENTITY 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82631 ; 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.error_if_exists: 0 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.create_if_missing: 0 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.paranoid_checks: 1 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.env: 0x55f889cc6860 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.fs: Posix File System 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.info_log: 0x55f88a595dc0 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_file_opening_threads: 16 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.statistics: (nil) 2024-06-21T00:36:55.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.use_fsync: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_log_file_size: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.keep_log_file_num: 1000 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.recycle_log_file_num: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_fallocate: 1 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_mmap_reads: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_mmap_writes: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.use_direct_reads: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.create_missing_column_families: 0 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.db_log_dir: 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-06-21T00:36:55.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.advise_random_on_open: 1 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.db_write_buffer_size: 0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.write_buffer_manager: 0x55f88a6862a0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-21T00:36:55.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.rate_limiter: (nil) 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.wal_recovery_mode: 2 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.enable_thread_tracking: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.enable_pipelined_write: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.unordered_write: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.row_cache: None 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.wal_filter: None 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_ingest_behind: 0 2024-06-21T00:36:55.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.preserve_deletes: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.two_write_queues: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.manual_wal_flush: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.atomic_flush: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.log_readahead_size: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.best_efforts_recovery: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.allow_data_in_errors: 0 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.db_host_id: __hostname__ 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_background_jobs: 2 2024-06-21T00:36:55.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_background_compactions: -1 2024-06-21T00:36:55.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_subcompactions: 1 2024-06-21T00:36:55.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-21T00:36:55.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-21T00:36:55.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-21T00:36:55.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_total_wal_size: 0 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_open_files: -1 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bytes_per_sync: 0 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-21T00:36:55.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_readahead_size: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_background_flushes: -1 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Compression algorithms supported: 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kZSTD supported: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kXpressCompression supported: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kLZ4HCCompression supported: 1 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kLZ4Compression supported: 1 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kBZip2Compression supported: 0 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kZlibCompression supported: 1 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: kSnappyCompression supported: 1 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-21T00:36:55.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.merge_operator: 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_filter: None 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_filter_factory: None 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.sst_partitioner_factory: None 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f88a561d00) 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-21T00:36:55.739 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_type: 0 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_index_type: 0 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_shortening: 1 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: hash_index_allow_collision: 1 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: checksum: 1 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: no_block_cache: 0 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache: 0x55f88a5cc170 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_options: 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: capacity : 536870912 2024-06-21T00:36:55.740 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_shard_bits : 4 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: strict_capacity_limit : 0 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_compressed: (nil) 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: persistent_cache: (nil) 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size: 4096 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size_deviation: 10 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_restart_interval: 16 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_block_restart_interval: 1 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_block_size: 4096 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: partition_filters: 0 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: use_delta_encoding: 1 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-21T00:36:55.741 INFO:journalctl@ceph.mon.a.smithi027.stdout: whole_key_filtering: 1 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout: verify_compression: 0 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout: format_version: 4 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_index_compression: 1 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_align: 0 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.write_buffer_size: 33554432 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_write_buffer_number: 2 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression: NoCompression 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression: Disabled 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.prefix_extractor: nullptr 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.num_levels: 7 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-21T00:36:55.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.level: 32767 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.strategy: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-21T00:36:55.743 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compression_opts.enabled: false 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.target_file_size_base: 67108864 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-21T00:36:55.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.arena_block_size: 4194304 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.disable_auto_compactions: 0 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-21T00:36:55.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.table_properties_collectors: 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.inplace_update_support: 0 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.bloom_locality: 0 2024-06-21T00:36:55.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.max_successive_merges: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.paranoid_file_checks: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.force_consistency_checks: 1 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.report_bg_io_stats: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.ttl: 2592000 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.enable_blob_files: false 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.min_blob_size: 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.blob_file_size: 268435456 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-21T00:36:55.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930215480941, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930215481699, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79181, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77444, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71753, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718930215, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "72f2d492-df0d-43bd-a951-2439928e1386", "db_session_id": "HIZ5JZSX3P56CY849KUF"}} 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930215482062, "job": 1, "event": "recovery_finished"} 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f88a5b2a80 2024-06-21T00:36:55.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: DB pointer 0x55f88a5c2000 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** DB Stats ** 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:36:55.749 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T00:36:55.750 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 23.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 23.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.751 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.0 0.00 0.00 1 0.000 0 0 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 23.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T00:36:55.752 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???) e1 preinit fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).mds e1 new map 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).mds e1 print_map 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: e1 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: legacy client fscid: -1 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout: No filesystems configured 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:36:55.753 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 bash[25117]: 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 systemd[1]: Started Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mon.a is new leader, mons a in quorum (ranks 0) 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: monmap e1: 1 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: fsmap 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: osdmap e1: 0 total, 0 up, 0 in 2024-06-21T00:36:55.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:55 smithi027 ceph-mon[25136]: mgrmap e1: no daemons active 2024-06-21T00:36:56.534 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote config to /etc/ceph/ceph.conf 2024-06-21T00:36:56.534 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-06-21T00:36:56.535 INFO:teuthology.orchestra.run.smithi027.stderr:Creating mgr... 2024-06-21T00:36:56.535 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying port 9283 ... 2024-06-21T00:36:56.714 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to reset failed state of unit ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Unit ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service not loaded. 2024-06-21T00:36:56.722 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968.target.wants/ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-47651940-2f66-11ef-bc9d-c7b262605968@.service. 2024-06-21T00:36:56.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:56 smithi027 systemd[1]: Starting Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:36:57.359 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:57 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3634119904' entity='client.admin' 2024-06-21T00:36:57.359 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 podman[25350]: 2024-06-21 00:36:57.016628545 +0000 UTC m=+0.019381028 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-06-21T00:36:57.359 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 podman[25350]: 2024-06-21 00:36:57.117093378 +0000 UTC m=+0.119845853 container create 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_CLEAN=True, version=8, name=centos-stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-21T00:36:57.360 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 podman[25350]: 2024-06-21 00:36:57.361243013 +0000 UTC m=+0.363995496 container init 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64, ceph=True, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-21T00:36:57.448 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-06-21T00:36:57.456 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:36:57.457 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-06-21T00:36:57.457 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to enable service . firewalld.service is not available 2024-06-21T00:36:57.466 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-06-21T00:36:57.473 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:36:57.475 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-06-21T00:36:57.475 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-06-21T00:36:57.475 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr to start... 2024-06-21T00:36:57.475 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr... 2024-06-21T00:36:57.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 podman[25350]: 2024-06-21 00:36:57.364412018 +0000 UTC m=+0.367164488 container start 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8) 2024-06-21T00:36:57.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 bash[25350]: 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 2024-06-21T00:36:57.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 systemd[1]: Started Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:36:57.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:57.515+0000 7f479bf0e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:36:57.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:57.651+0000 7f479bf0e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:36:57.970 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:57 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:57.755+0000 7f479bf0e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:36:57.973 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:36:57.973 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "47651940-2f66-11ef-bc9d-c7b262605968", 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-21T00:36:57.974 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-21T00:36:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-21T00:36:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": false, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "nfs", 2024-06-21T00:36:57.980 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-06-21T00:36:51.034748+0000", 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-21T00:36:57.981 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:36:58.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:36:58 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/690949969' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-21T00:36:58.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:58.092+0000 7f479bf0e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:36:58.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:58.178+0000 7f479bf0e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:36:58.486 INFO:teuthology.orchestra.run.smithi027.stderr:mgr not available, waiting (1/15)... 2024-06-21T00:36:59.622 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:59.371+0000 7f479bf0e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:36:59.622 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:59.539+0000 7f479bf0e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:36:59.622 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:59.622+0000 7f479bf0e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:37:00.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:36:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:36:59.943+0000 7f479bf0e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:37:00.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:00.036+0000 7f479bf0e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:37:00.230 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:00.130+0000 7f479bf0e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:37:00.715 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:00.378+0000 7f479bf0e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:37:00.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:00.851+0000 7f479bf0e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "47651940-2f66-11ef-bc9d-c7b262605968", 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-06-21T00:37:01.005 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-06-21T00:37:01.006 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-06-21T00:37:01.007 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-21T00:37:01.008 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-21T00:37:01.009 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.010 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": false, 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "nfs", 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:37:01.011 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-06-21T00:36:51.034748+0000", 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-21T00:37:01.012 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:01.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:01 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3510167896' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-21T00:37:01.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:01.361+0000 7f479bf0e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:37:01.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:01.458+0000 7f479bf0e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:37:02.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:01.765+0000 7f479bf0e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:37:02.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:01.856+0000 7f479bf0e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:37:02.621 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:02.272+0000 7f479bf0e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:37:02.621 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:02.409+0000 7f479bf0e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:37:02.621 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:02.504+0000 7f479bf0e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:37:02.621 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:02.621+0000 7f479bf0e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:37:02.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:02.712+0000 7f479bf0e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:37:03.708 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:03.287+0000 7f479bf0e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:37:03.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:03.658+0000 7f479bf0e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:37:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: Activating manager daemon y 2024-06-21T00:37:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: mgrmap e2: y(active, starting, since 0.00274325s) 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:37:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:03 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: from='mgr.14100 172.21.15.27:0/3462335796' entity='mgr.y' 2024-06-21T00:37:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:04 smithi027 ceph-mon[25136]: mgrmap e3: y(active, since 1.00642s) 2024-06-21T00:37:06.844 INFO:teuthology.orchestra.run.smithi027.stderr:mgr not available, waiting (2/15)... 2024-06-21T00:37:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:07 smithi027 ceph-mon[25136]: mgrmap e4: y(active, since 2s) 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "47651940-2f66-11ef-bc9d-c7b262605968", 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-06-21T00:37:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-06-21T00:37:09.529 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:09.529 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-21T00:37:09.529 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-21T00:37:09.529 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-21T00:37:09.529 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-21T00:37:09.532 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "nfs", 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-06-21T00:36:51.034748+0000", 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-06-21T00:37:09.534 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-06-21T00:37:09.535 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-21T00:37:09.535 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2443361907' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-21T00:37:10.178 INFO:teuthology.orchestra.run.smithi027.stderr:mgr is available 2024-06-21T00:37:10.759 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:37:10.759 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [global] 2024-06-21T00:37:10.759 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: fsid = 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [mgr] 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [osd] 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-21T00:37:10.760 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-21T00:37:11.225 INFO:teuthology.orchestra.run.smithi027.stderr:Enabling cephadm module... 2024-06-21T00:37:11.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1519375631' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-21T00:37:11.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1519375631' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-06-21T00:37:12.806 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4025222509' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-06-21T00:37:12.806 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setuser ceph since I am not root 2024-06-21T00:37:12.806 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setgroup ceph since I am not root 2024-06-21T00:37:12.806 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:12.669+0000 7fe84fee4000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:37:12.806 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:12.806+0000 7fe84fee4000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:37:13.112 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:12.909+0000 7fe84fee4000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:37:13.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:13.237+0000 7fe84fee4000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:37:13.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:13.325+0000 7fe84fee4000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:37:13.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:13 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4025222509' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-06-21T00:37:13.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:13 smithi027 ceph-mon[25136]: mgrmap e5: y(active, since 8s) 2024-06-21T00:37:14.773 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:14.517+0000 7fe84fee4000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:37:14.773 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:14.688+0000 7fe84fee4000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:37:14.773 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:14.774+0000 7fe84fee4000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:37:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:15 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/481350130' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-21T00:37:15.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:15 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:15.105+0000 7fe84fee4000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:37:15.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:15 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:15.199+0000 7fe84fee4000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:37:15.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:15 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:15.304+0000 7fe84fee4000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:37:15.895 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:15 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:15.557+0000 7fe84fee4000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 5, 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "active_name": "y", 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-21T00:37:15.896 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:16.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:16 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:16.021+0000 7fe84fee4000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:37:16.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:16 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:16.529+0000 7fe84fee4000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:37:16.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:16 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:16.626+0000 7fe84fee4000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:37:16.725 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for the mgr to restart... 2024-06-21T00:37:16.725 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr epoch 5... 2024-06-21T00:37:16.974 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:16 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:16.939+0000 7fe84fee4000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:37:17.462 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.037+0000 7fe84fee4000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:37:17.462 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.463+0000 7fe84fee4000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:37:17.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.599+0000 7fe84fee4000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:37:17.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.695+0000 7fe84fee4000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:37:18.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.812+0000 7fe84fee4000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:37:18.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:17.904+0000 7fe84fee4000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:37:18.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:18.474+0000 7fe84fee4000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:37:19.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:18.848+0000 7fe84fee4000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:37:19.850 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:19] ENGINE Bus STARTING 2024-06-21T00:37:19.850 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:19] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:37:19.851 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:19] ENGINE Bus STARTED 2024-06-21T00:37:19.911 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:19.911 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-06-21T00:37:19.911 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "initialized": true 2024-06-21T00:37:19.912 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:20.221 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: Active manager daemon y restarted 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: Activating manager daemon y 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: osdmap e2: 0 total, 0 up, 0 in 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: mgrmap e6: y(active, starting, since 0.0523111s) 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:20.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:20.223 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:37:20.223 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:37:20.223 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:20.223 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:19 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:20.524 INFO:teuthology.orchestra.run.smithi027.stderr:mgr epoch 5 is available 2024-06-21T00:37:20.524 INFO:teuthology.orchestra.run.smithi027.stderr:Setting orchestrator backend to cephadm... 2024-06-21T00:37:21.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:19] ENGINE Bus STARTING 2024-06-21T00:37:21.165 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:19] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:37:21.165 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:19] ENGINE Bus STARTED 2024-06-21T00:37:21.165 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-06-21T00:37:21.165 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: mgrmap e7: y(active, since 1.05503s) 2024-06-21T00:37:21.165 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:20 smithi027 ceph-mon[25136]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-06-21T00:37:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:22 smithi027 ceph-mon[25136]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:22 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:22 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:22 smithi027 ceph-mon[25136]: mgrmap e8: y(active, since 2s) 2024-06-21T00:37:22.242 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: value unchanged 2024-06-21T00:37:22.772 INFO:teuthology.orchestra.run.smithi027.stderr:Generating ssh key... 2024-06-21T00:37:23.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-mon[25136]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:24 smithi027 ceph-mon[25136]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:24 smithi027 ceph-mon[25136]: Generating ssh key... 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:24 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:24 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: Generating public/private rsa key pair. 2024-06-21T00:37:24.278 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: Your identification has been saved in /tmp/tmpln8edmef/key. 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: Your public key has been saved in /tmp/tmpln8edmef/key.pub. 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: The key fingerprint is: 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: SHA256:GV5NCGsthKMH6nBjJvSAM8dm/aQ7WYFyWizNeOe93V0 ceph-47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: The key's randomart image is: 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: +---[RSA 3072]----+ 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: |.. * . .o. .. | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: |+oO @ *. o.o | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: |.*oO O += o . | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: |o B.o =o.= E| 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: | B . = So . . . | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: | . + . . . . | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: | . | 2024-06-21T00:37:24.279 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: | | 2024-06-21T00:37:24.280 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: | | 2024-06-21T00:37:24.280 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: +----[SHA256]-----+ 2024-06-21T00:37:25.148 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC15gUfO5gSXe24un/moXauRZpbZXAQ3LLjwBL3FBJlzAEk2Z0bT2SJsqxYs2NTvNDxTtm6m1wE8s0XU80z0+o5YJYZ6LioQ6eKjRgobXNj6ZicGMrWszXeVWA6yt/rCUhZFjMsB1L3GCHbFl+aKrJ9cmzY19+5HaOPFs/Hf/n004LMbtZ/9fom09Xwju/Gd5lEDhtzQUi7FDMVjTAmLWfpzRKVKDfDlTO1b7HOFCV/1bsMMuLCaJiodbIRYaSiRQ1PP1YXQDg5pJaVbtGjLU+wy1f19sm3KapC9YCTW99L0MHq+wSx3VySsAJpCMProZk9l5/xMFHYbMLwAxQGkZlMZ+gugM1t3jofacCVKsAnS48RsGGdNKwXCaaaEPqMkEMVi+LTuKIRDHDCVMhfTV3EvYo64ir49dPZFB9jtZCHi7kyDy79QKGCP5spX9EazlIA2fZZXTkh73qTmvY5sSHTR+lMShzErheWtHfr90KqQImu4NiwSdjF0V8QfaTyKzs= ceph-47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:37:25.667 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-06-21T00:37:25.668 INFO:teuthology.orchestra.run.smithi027.stderr:Adding key to root@localhost authorized_keys... 2024-06-21T00:37:25.668 INFO:teuthology.orchestra.run.smithi027.stderr:Adding host smithi027... 2024-06-21T00:37:26.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:26 smithi027 ceph-mon[25136]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:27.191 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Added host 'smithi027' with addr '172.21.15.27' 2024-06-21T00:37:27.728 INFO:teuthology.orchestra.run.smithi027.stderr:Deploying unmanaged mon service... 2024-06-21T00:37:27.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:27 smithi027 ceph-mon[25136]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi027", "addr": "172.21.15.27", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:27.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:27 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:27.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:27 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:28.366 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Scheduled mon update... 2024-06-21T00:37:28.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:28 smithi027 ceph-mon[25136]: Deploying cephadm binary to smithi027 2024-06-21T00:37:28.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:28 smithi027 ceph-mon[25136]: Added host smithi027 2024-06-21T00:37:28.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:28 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:28.939 INFO:teuthology.orchestra.run.smithi027.stderr:Deploying unmanaged mgr service... 2024-06-21T00:37:29.621 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-06-21T00:37:29.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:29 smithi027 ceph-mon[25136]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:29.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:29 smithi027 ceph-mon[25136]: Saving service mon spec with placement count:5 2024-06-21T00:37:30.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:30 smithi027 ceph-mon[25136]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:30.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:30 smithi027 ceph-mon[25136]: Saving service mgr spec with placement count:2 2024-06-21T00:37:30.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:30 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:31.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:31 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:31.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:31 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:32.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:32 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3384708875' entity='client.admin' 2024-06-21T00:37:33.911 INFO:teuthology.orchestra.run.smithi027.stderr:Enabling the dashboard module... 2024-06-21T00:37:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:34 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1463500535' entity='client.admin' 2024-06-21T00:37:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:34 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:35.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4243012248' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-06-21T00:37:35.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-mon[25136]: from='mgr.14118 172.21.15.27:0/3226855169' entity='mgr.y' 2024-06-21T00:37:35.378 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setuser ceph since I am not root 2024-06-21T00:37:35.378 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setgroup ceph since I am not root 2024-06-21T00:37:35.378 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:35.244+0000 7f973251d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:37:35.378 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:35.377+0000 7f973251d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:37:35.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:35.481+0000 7f973251d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:37:36.073 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:35.803+0000 7f973251d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:37:36.073 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:35.891+0000 7f973251d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:37:36.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:36 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4243012248' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-06-21T00:37:36.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:36 smithi027 ceph-mon[25136]: mgrmap e9: y(active, since 16s) 2024-06-21T00:37:36.878 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:36.879 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 9, 2024-06-21T00:37:36.879 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-06-21T00:37:36.879 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "active_name": "y", 2024-06-21T00:37:36.879 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-21T00:37:36.879 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:37.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2046875507' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-21T00:37:37.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.088+0000 7f973251d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:37:37.541 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.258+0000 7f973251d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:37:37.542 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.343+0000 7f973251d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:37:37.763 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for the mgr to restart... 2024-06-21T00:37:37.763 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr epoch 9... 2024-06-21T00:37:37.862 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.670+0000 7f973251d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:37:37.863 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.763+0000 7f973251d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:37:37.863 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:37.863+0000 7f973251d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:37:38.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:38.120+0000 7f973251d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:37:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:38.583+0000 7f973251d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:37:39.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:39.085+0000 7f973251d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:37:39.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:39.182+0000 7f973251d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:37:39.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:39.488+0000 7f973251d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:37:39.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:39.580+0000 7f973251d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:37:40.344 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:39.995+0000 7f973251d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:37:40.344 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:40.132+0000 7f973251d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:37:40.344 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:40.227+0000 7f973251d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:37:40.345 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:40.345+0000 7f973251d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:37:40.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:40.436+0000 7f973251d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:37:41.376 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:41.006+0000 7f973251d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:37:41.678 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:37:41.377+0000 7f973251d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:37:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: Active manager daemon y restarted 2024-06-21T00:37:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: Activating manager daemon y 2024-06-21T00:37:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: osdmap e3: 0 total, 0 up, 0 in 2024-06-21T00:37:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: mgrmap e10: y(active, starting, since 0.115861s) 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:42.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:37:42.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:37:42.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-06-21T00:37:42.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-06-21T00:37:42.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "initialized": true 2024-06-21T00:37:42.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-06-21T00:37:42.804 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:42] ENGINE Bus STARTING 2024-06-21T00:37:42.804 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:42] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:37:43.088 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:37:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:37:42] ENGINE Bus STARTED 2024-06-21T00:37:43.090 INFO:teuthology.orchestra.run.smithi027.stderr:mgr epoch 9 is available 2024-06-21T00:37:43.090 INFO:teuthology.orchestra.run.smithi027.stderr:Generating a dashboard self-signed certificate... 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: mgrmap e11: y(active, since 1.11835s) 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:42] ENGINE Bus STARTING 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:42] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: [21/Jun/2024:00:37:42] ENGINE Bus STARTED 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:43.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:43.927 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Self-signed certificate created 2024-06-21T00:37:44.514 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial admin user... 2024-06-21T00:37:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:44 smithi027 ceph-mon[25136]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:44 smithi027 ceph-mon[25136]: mgrmap e12: y(active, since 2s) 2024-06-21T00:37:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:45.438 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$zmTFQXX60QFMuEkDQ.WuW.BjyAjp5I9EGn2S0u8e9kkLaBGfWRnR2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1718930265, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-06-21T00:37:46.327 INFO:teuthology.orchestra.run.smithi027.stderr:Fetching dashboard port number... 2024-06-21T00:37:46.581 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:46 smithi027 ceph-mon[25136]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:46.581 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:46 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:46.984 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 8443 2024-06-21T00:37:47.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:47 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2935460841' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-06-21T00:37:47.857 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-06-21T00:37:47.865 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-06-21T00:37:47.866 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-06-21T00:37:47.866 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr:Ceph Dashboard is now available at: 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr: URL: https://smithi027.front.sepia.ceph.com:8443/ 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr: User: admin 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr: Password: jsr1b0hbwl 2024-06-21T00:37:47.869 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:47.870 INFO:teuthology.orchestra.run.smithi027.stderr:Enabling autotune for osd_memory_target 2024-06-21T00:37:48.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:48 smithi027 ceph-mon[25136]: mgrmap e13: y(active, since 6s) 2024-06-21T00:37:49.862 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr:You can access the Ceph CLI with: 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr:Please consider enabling telemetry to help improve Ceph: 2024-06-21T00:37:50.493 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr: ceph telemetry on 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr:For more information see: 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T00:37:50.494 INFO:teuthology.orchestra.run.smithi027.stderr:Bootstrap complete. 2024-06-21T00:37:50.533 INFO:tasks.cephadm:Fetching config... 2024-06-21T00:37:50.534 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:37:50.534 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-06-21T00:37:50.552 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-06-21T00:37:50.553 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:37:50.553 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-06-21T00:37:50.609 INFO:tasks.cephadm:Fetching mon keyring... 2024-06-21T00:37:50.609 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:37:50.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/keyring of=/dev/stdout 2024-06-21T00:37:50.674 INFO:tasks.cephadm:Fetching pub ssh key... 2024-06-21T00:37:50.675 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:37:50.675 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-06-21T00:37:50.731 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-06-21T00:37:50.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC15gUfO5gSXe24un/moXauRZpbZXAQ3LLjwBL3FBJlzAEk2Z0bT2SJsqxYs2NTvNDxTtm6m1wE8s0XU80z0+o5YJYZ6LioQ6eKjRgobXNj6ZicGMrWszXeVWA6yt/rCUhZFjMsB1L3GCHbFl+aKrJ9cmzY19+5HaOPFs/Hf/n004LMbtZ/9fom09Xwju/Gd5lEDhtzQUi7FDMVjTAmLWfpzRKVKDfDlTO1b7HOFCV/1bsMMuLCaJiodbIRYaSiRQ1PP1YXQDg5pJaVbtGjLU+wy1f19sm3KapC9YCTW99L0MHq+wSx3VySsAJpCMProZk9l5/xMFHYbMLwAxQGkZlMZ+gugM1t3jofacCVKsAnS48RsGGdNKwXCaaaEPqMkEMVi+LTuKIRDHDCVMhfTV3EvYo64ir49dPZFB9jtZCHi7kyDy79QKGCP5spX9EazlIA2fZZXTkh73qTmvY5sSHTR+lMShzErheWtHfr90KqQImu4NiwSdjF0V8QfaTyKzs= ceph-47651940-2f66-11ef-bc9d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-21T00:37:50.813 INFO:teuthology.orchestra.run.smithi027.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC15gUfO5gSXe24un/moXauRZpbZXAQ3LLjwBL3FBJlzAEk2Z0bT2SJsqxYs2NTvNDxTtm6m1wE8s0XU80z0+o5YJYZ6LioQ6eKjRgobXNj6ZicGMrWszXeVWA6yt/rCUhZFjMsB1L3GCHbFl+aKrJ9cmzY19+5HaOPFs/Hf/n004LMbtZ/9fom09Xwju/Gd5lEDhtzQUi7FDMVjTAmLWfpzRKVKDfDlTO1b7HOFCV/1bsMMuLCaJiodbIRYaSiRQ1PP1YXQDg5pJaVbtGjLU+wy1f19sm3KapC9YCTW99L0MHq+wSx3VySsAJpCMProZk9l5/xMFHYbMLwAxQGkZlMZ+gugM1t3jofacCVKsAnS48RsGGdNKwXCaaaEPqMkEMVi+LTuKIRDHDCVMhfTV3EvYo64ir49dPZFB9jtZCHi7kyDy79QKGCP5spX9EazlIA2fZZXTkh73qTmvY5sSHTR+lMShzErheWtHfr90KqQImu4NiwSdjF0V8QfaTyKzs= ceph-47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:37:50.831 DEBUG:teuthology.orchestra.run.smithi120:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC15gUfO5gSXe24un/moXauRZpbZXAQ3LLjwBL3FBJlzAEk2Z0bT2SJsqxYs2NTvNDxTtm6m1wE8s0XU80z0+o5YJYZ6LioQ6eKjRgobXNj6ZicGMrWszXeVWA6yt/rCUhZFjMsB1L3GCHbFl+aKrJ9cmzY19+5HaOPFs/Hf/n004LMbtZ/9fom09Xwju/Gd5lEDhtzQUi7FDMVjTAmLWfpzRKVKDfDlTO1b7HOFCV/1bsMMuLCaJiodbIRYaSiRQ1PP1YXQDg5pJaVbtGjLU+wy1f19sm3KapC9YCTW99L0MHq+wSx3VySsAJpCMProZk9l5/xMFHYbMLwAxQGkZlMZ+gugM1t3jofacCVKsAnS48RsGGdNKwXCaaaEPqMkEMVi+LTuKIRDHDCVMhfTV3EvYo64ir49dPZFB9jtZCHi7kyDy79QKGCP5spX9EazlIA2fZZXTkh73qTmvY5sSHTR+lMShzErheWtHfr90KqQImu4NiwSdjF0V8QfaTyKzs= ceph-47651940-2f66-11ef-bc9d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-21T00:37:50.874 INFO:teuthology.orchestra.run.smithi120.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC15gUfO5gSXe24un/moXauRZpbZXAQ3LLjwBL3FBJlzAEk2Z0bT2SJsqxYs2NTvNDxTtm6m1wE8s0XU80z0+o5YJYZ6LioQ6eKjRgobXNj6ZicGMrWszXeVWA6yt/rCUhZFjMsB1L3GCHbFl+aKrJ9cmzY19+5HaOPFs/Hf/n004LMbtZ/9fom09Xwju/Gd5lEDhtzQUi7FDMVjTAmLWfpzRKVKDfDlTO1b7HOFCV/1bsMMuLCaJiodbIRYaSiRQ1PP1YXQDg5pJaVbtGjLU+wy1f19sm3KapC9YCTW99L0MHq+wSx3VySsAJpCMProZk9l5/xMFHYbMLwAxQGkZlMZ+gugM1t3jofacCVKsAnS48RsGGdNKwXCaaaEPqMkEMVi+LTuKIRDHDCVMhfTV3EvYo64ir49dPZFB9jtZCHi7kyDy79QKGCP5spX9EazlIA2fZZXTkh73qTmvY5sSHTR+lMShzErheWtHfr90KqQImu4NiwSdjF0V8QfaTyKzs= ceph-47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:37:50.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-06-21T00:37:51.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:50 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1529316006' entity='client.admin' 2024-06-21T00:37:53.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:53.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:37:53.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:53.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:53.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:55.083 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-06-21T00:37:55.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-06-21T00:37:55.704 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:55 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/42325094' entity='client.admin' 2024-06-21T00:37:55.704 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:55.704 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:57.630 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi120 2024-06-21T00:37:57.631 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:37:57.631 DEBUG:teuthology.orchestra.run.smithi120:> dd of=/etc/ceph/ceph.conf 2024-06-21T00:37:57.649 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:37:57.649 DEBUG:teuthology.orchestra.run.smithi120:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:37:57.708 INFO:tasks.cephadm:Adding host smithi120 to orchestrator... 2024-06-21T00:37:57.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch host add smithi120 2024-06-21T00:37:58.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:37:58.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:58.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:37:58.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:37:58.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:37:58.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:37:58.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:58.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:37:59.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:37:58 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:38:00.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:00 smithi027 ceph-mon[25136]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi120", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:00.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:00 smithi027 ceph-mon[25136]: Deploying cephadm binary to smithi120 2024-06-21T00:38:00.540 INFO:teuthology.orchestra.run.smithi027.stdout:Added host 'smithi120' with addr '172.21.15.120' 2024-06-21T00:38:01.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch host ls --format=json 2024-06-21T00:38:01.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:01.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: Added host smithi120 2024-06-21T00:38:01.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:01.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:01.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:01.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:01.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:02.908 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:38:02.908 INFO:teuthology.orchestra.run.smithi027.stdout:[{"addr": "172.21.15.27", "hostname": "smithi027", "labels": [], "status": ""}, {"addr": "172.21.15.120", "hostname": "smithi120", "labels": [], "status": ""}] 2024-06-21T00:38:03.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:03 smithi027 ceph-mon[25136]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:03.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:03.408 INFO:tasks.cephadm:Setting crush tunables to default 2024-06-21T00:38:03.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd crush tunables default 2024-06-21T00:38:04.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:04 smithi027 ceph-mon[25136]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:38:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:05 smithi027 ceph-mon[25136]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:05 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:05 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:05 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi120", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:38:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:05 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:06.122 INFO:teuthology.orchestra.run.smithi027.stderr:adjusted tunables profile to default 2024-06-21T00:38:06.437 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:06 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/352834435' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-21T00:38:06.724 INFO:tasks.cephadm:Adding mon.a on smithi027 2024-06-21T00:38:06.725 INFO:tasks.cephadm:Adding mon.c on smithi027 2024-06-21T00:38:06.725 INFO:tasks.cephadm:Adding mon.b on smithi120 2024-06-21T00:38:06.725 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply mon '3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b' 2024-06-21T00:38:07.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:07 smithi027 ceph-mon[25136]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:07.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:07 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/352834435' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-21T00:38:07.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:07 smithi027 ceph-mon[25136]: osdmap e4: 0 total, 0 up, 0 in 2024-06-21T00:38:08.506 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled mon update... 2024-06-21T00:38:09.144 DEBUG:teuthology.orchestra.run.smithi027:mon.c> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.c.service 2024-06-21T00:38:09.147 DEBUG:teuthology.orchestra.run.smithi120:mon.b> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.b.service 2024-06-21T00:38:09.149 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-21T00:38:09.150 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:09.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:09.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:09.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:10.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:10 smithi027 ceph-mon[25136]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:10.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:10 smithi027 ceph-mon[25136]: Saving service mon spec with placement smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b;count:3 2024-06-21T00:38:10.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:10 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:38:10.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:10 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:38:10.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:10 smithi027 ceph-mon[25136]: Deploying daemon mon.c on smithi027 2024-06-21T00:38:11.168 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:38:11.168 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":1,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","modified":"2024-06-21T00:36:48.093788Z","created":"2024-06-21T00:36:48.093788Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-21T00:38:11.170 INFO:teuthology.orchestra.run.smithi120.stderr:dumped monmap epoch 1 2024-06-21T00:38:11.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:11 smithi027 ceph-mon[25136]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:11.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:11 smithi027 systemd[1]: Starting Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:38:12.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:11 smithi027 podman[29795]: 2024-06-21 00:38:11.709159234 +0000 UTC m=+0.018047232 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-21T00:38:12.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:11 smithi027 podman[29795]: 2024-06-21 00:38:11.810867578 +0000 UTC m=+0.119755561 container create 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, release=754, GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public) 2024-06-21T00:38:12.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 podman[29795]: 2024-06-21 00:38:12.021548979 +0000 UTC m=+0.330436977 container init 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD) 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 podman[29795]: 2024-06-21 00:38:12.025800069 +0000 UTC m=+0.334688065 container start 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, release=754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, version=8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: set uid:gid to 167:167 (ceph:ceph) 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: pidfile_write: ignore empty --pid-file 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: load: jerasure load: lrc 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: RocksDB version: 6.15.5 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Compile date Apr 18 2022 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: DB SUMMARY 2024-06-21T00:38:12.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: DB Session ID: WAQL7U2WTT1WJHVLQFNA 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: CURRENT file: CURRENT 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: IDENTITY file: IDENTITY 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.error_if_exists: 0 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.create_if_missing: 0 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.paranoid_checks: 1 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.env: 0x5613722c6860 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.fs: Posix File System 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.info_log: 0x561373601dc0 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_file_opening_threads: 16 2024-06-21T00:38:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.statistics: (nil) 2024-06-21T00:38:12.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.use_fsync: 0 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_log_file_size: 0 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.keep_log_file_num: 1000 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.recycle_log_file_num: 0 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_fallocate: 1 2024-06-21T00:38:12.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_mmap_reads: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_mmap_writes: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.use_direct_reads: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.create_missing_column_families: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.db_log_dir: 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.advise_random_on_open: 1 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.db_write_buffer_size: 0 2024-06-21T00:38:12.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.write_buffer_manager: 0x5613736f22d0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.rate_limiter: (nil) 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.wal_recovery_mode: 2 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.enable_thread_tracking: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.enable_pipelined_write: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.unordered_write: 0 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-21T00:38:12.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.row_cache: None 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.wal_filter: None 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_ingest_behind: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.preserve_deletes: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.two_write_queues: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.manual_wal_flush: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.atomic_flush: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.log_readahead_size: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.best_efforts_recovery: 0 2024-06-21T00:38:12.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.allow_data_in_errors: 0 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.db_host_id: __hostname__ 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_background_jobs: 2 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_background_compactions: -1 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_subcompactions: 1 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_total_wal_size: 0 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-21T00:38:12.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_open_files: -1 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bytes_per_sync: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_readahead_size: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_background_flushes: -1 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Compression algorithms supported: 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kZSTD supported: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kXpressCompression supported: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kLZ4HCCompression supported: 1 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kLZ4Compression supported: 1 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kBZip2Compression supported: 0 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kZlibCompression supported: 1 2024-06-21T00:38:12.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: kSnappyCompression supported: 1 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.merge_operator: 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_filter: None 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_filter_factory: None 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.sst_partitioner_factory: None 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5613735cdd00) 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-21T00:38:12.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_type: 0 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_index_type: 0 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_shortening: 1 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: hash_index_allow_collision: 1 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: checksum: 1 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: no_block_cache: 0 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache: 0x561373638170 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_options: 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: capacity : 536870912 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_shard_bits : 4 2024-06-21T00:38:12.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: strict_capacity_limit : 0 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_compressed: (nil) 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: persistent_cache: (nil) 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size: 4096 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size_deviation: 10 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_restart_interval: 16 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_block_restart_interval: 1 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_block_size: 4096 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: partition_filters: 0 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: use_delta_encoding: 1 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-21T00:38:12.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: whole_key_filtering: 1 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: verify_compression: 0 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: format_version: 4 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_index_compression: 1 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_align: 0 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.write_buffer_size: 33554432 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_write_buffer_number: 2 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression: NoCompression 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression: Disabled 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.prefix_extractor: nullptr 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.num_levels: 7 2024-06-21T00:38:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.level: 32767 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.strategy: 0 2024-06-21T00:38:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compression_opts.enabled: false 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.target_file_size_base: 67108864 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-21T00:38:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.arena_block_size: 4194304 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.disable_auto_compactions: 0 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-21T00:38:12.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.table_properties_collectors: 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.inplace_update_support: 0 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-21T00:38:12.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.bloom_locality: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.max_successive_merges: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.paranoid_file_checks: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.force_consistency_checks: 1 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.report_bg_io_stats: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.ttl: 2592000 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.enable_blob_files: false 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.min_blob_size: 0 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.blob_file_size: 268435456 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-21T00:38:12.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930292053692, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930292054022, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718930292, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "355edfb9-f552-4bee-8a82-9776fe3c64fc", "db_session_id": "WAQL7U2WTT1WJHVLQFNA"}} 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718930292054384, "job": 1, "event": "recovery_finished"} 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56137361ea80 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: DB pointer 0x56137362e000 2024-06-21T00:38:12.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** DB Stats ** 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:38:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T00:38:12.076 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 0.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 0.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T00:38:12.077 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T00:38:12.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-06-21T00:38:12.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: using public_addrv [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] 2024-06-21T00:38:12.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: starting mon.c rank -1 at public addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] at bind addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:38:12.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(???) e0 preinit fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:38:12.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).mds e1 new map 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).mds e1 print_map 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: e1 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: legacy client fscid: -1 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout: No filesystems configured 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-06-21T00:38:12.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:12.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: Saving service mon spec with placement smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b;count:3 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: Deploying daemon mon.c on smithi027 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 ceph-mon[29814]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 bash[29795]: 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 2024-06-21T00:38:12.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:12 smithi027 systemd[1]: Started Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:38:12.736 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-21T00:38:12.736 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: mon.a calling monitor election 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: mon.c calling monitor election 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: fsmap 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: osdmap e4: 0 total, 0 up, 0 in 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: mgrmap e13: y(active, since 35s) 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:17.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[25136]: Deploying daemon mon.b on smithi120 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: mon.a calling monitor election 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: mon.c calling monitor election 2024-06-21T00:38:17.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: fsmap 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: osdmap e4: 0 total, 0 up, 0 in 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: mgrmap e13: y(active, since 35s) 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:17.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:17 smithi027 ceph-mon[29814]: Deploying daemon mon.b on smithi120 2024-06-21T00:38:17.497 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:38:17.497 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":2,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","modified":"2024-06-21T00:38:12.075259Z","created":"2024-06-21T00:36:48.093788Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-21T00:38:17.499 INFO:teuthology.orchestra.run.smithi120.stderr:dumped monmap epoch 2 2024-06-21T00:38:18.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/1157576973' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:18.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[25136]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:18.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:18.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/1157576973' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:18.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[29814]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:18.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:18 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:19.214 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-21T00:38:19.215 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-21T00:38:20.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:20 smithi027 ceph-mon[25136]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:20.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:20 smithi027 ceph-mon[29814]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:21.942 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:38:21.942 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":2,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","modified":"2024-06-21T00:38:12.075259Z","created":"2024-06-21T00:36:48.093788Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-21T00:38:21.943 INFO:teuthology.orchestra.run.smithi120.stderr:dumped monmap epoch 2 2024-06-21T00:38:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:22 smithi027 ceph-mon[25136]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:22 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/2834905654' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:22 smithi027 ceph-mon[29814]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:22 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/2834905654' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:23.469 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-06-21T00:38:23.469 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph mon dump -f json 2024-06-21T00:38:28.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:38:28.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: mon.a calling monitor election 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: mon.c calling monitor election 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: mon.b calling monitor election 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T00:38:28.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: fsmap 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: osdmap e4: 0 total, 0 up, 0 in 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: mgrmap e13: y(active, since 46s) 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: mon.a calling monitor election 2024-06-21T00:38:28.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: mon.c calling monitor election 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: mon.b calling monitor election 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: fsmap 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: osdmap e4: 0 total, 0 up, 0 in 2024-06-21T00:38:28.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: mgrmap e13: y(active, since 46s) 2024-06-21T00:38:28.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T00:38:28.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:28.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:28.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:28.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:28.324 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-06-21T00:38:28.325 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":3,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","modified":"2024-06-21T00:38:22.715626Z","created":"2024-06-21T00:36:48.093788Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:3300","nonce":0},{"type":"v1","addr":"172.21.15.120:6789","nonce":0}]},"addr":"172.21.15.120:6789/0","public_addr":"172.21.15.120:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-06-21T00:38:28.326 INFO:teuthology.orchestra.run.smithi120.stderr:dumped monmap epoch 3 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[25136]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/2878413696' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[29814]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/2878413696' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:38:29.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:28 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.269 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-06-21T00:38:29.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph config generate-minimal-conf 2024-06-21T00:38:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: Reconfiguring mon.a (unknown last config time)... 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:29.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[25136]: Reconfiguring daemon mon.a on smithi027 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:29.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: Reconfiguring mon.a (unknown last config time)... 2024-06-21T00:38:29.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:29.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:29.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:29.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:29 smithi027 ceph-mon[29814]: Reconfiguring daemon mon.a on smithi027 2024-06-21T00:38:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:29 smithi120 ceph-mon[25975]: Reconfiguring daemon mon.a on smithi027 2024-06-21T00:38:31.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[29814]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:31.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[25136]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:31.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:31 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:31 smithi120 ceph-mon[25975]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:31 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:31 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:31 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:31 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[25136]: Reconfiguring mon.c (monmap changed)... 2024-06-21T00:38:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[25136]: Reconfiguring daemon mon.c on smithi027 2024-06-21T00:38:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:32.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[29814]: Reconfiguring mon.c (monmap changed)... 2024-06-21T00:38:32.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[29814]: Reconfiguring daemon mon.c on smithi027 2024-06-21T00:38:32.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:32 smithi120 ceph-mon[25975]: Reconfiguring mon.c (monmap changed)... 2024-06-21T00:38:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:32 smithi120 ceph-mon[25975]: Reconfiguring daemon mon.c on smithi027 2024-06-21T00:38:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:32.612 INFO:teuthology.orchestra.run.smithi027.stdout:# minimal ceph.conf for 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:38:32.612 INFO:teuthology.orchestra.run.smithi027.stdout:[global] 2024-06-21T00:38:32.613 INFO:teuthology.orchestra.run.smithi027.stdout: fsid = 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:38:32.613 INFO:teuthology.orchestra.run.smithi027.stdout: mon_host = [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2637189729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2637189729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.364 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:33.364 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2637189729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.364 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:33.364 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T00:38:33.365 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T00:38:33.365 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:33.458 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-06-21T00:38:33.459 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:38:33.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.conf 2024-06-21T00:38:33.487 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:38:33.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:38:33.553 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:38:33.553 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd of=/etc/ceph/ceph.conf 2024-06-21T00:38:33.581 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:38:33.582 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:38:33.647 INFO:tasks.cephadm:Adding mgr.y on smithi027 2024-06-21T00:38:33.648 INFO:tasks.cephadm:Adding mgr.x on smithi120 2024-06-21T00:38:33.648 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply mgr '2;smithi027=y;smithi120=x' 2024-06-21T00:38:34.272 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:34 smithi120 ceph-mon[25975]: Reconfiguring mon.b (monmap changed)... 2024-06-21T00:38:34.272 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:34 smithi120 ceph-mon[25975]: Reconfiguring daemon mon.b on smithi120 2024-06-21T00:38:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:34 smithi027 ceph-mon[25136]: Reconfiguring mon.b (monmap changed)... 2024-06-21T00:38:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:34 smithi027 ceph-mon[25136]: Reconfiguring daemon mon.b on smithi120 2024-06-21T00:38:34.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:34 smithi027 ceph-mon[29814]: Reconfiguring mon.b (monmap changed)... 2024-06-21T00:38:34.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:34 smithi027 ceph-mon[29814]: Reconfiguring daemon mon.b on smithi120 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:35.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:35.947 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled mgr update... 2024-06-21T00:38:36.480 DEBUG:teuthology.orchestra.run.smithi120:mgr.x> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service 2024-06-21T00:38:36.483 INFO:tasks.cephadm:Deploying OSDs... 2024-06-21T00:38:36.483 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:38:36.483 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/scratch_devs of=/dev/stdout 2024-06-21T00:38:36.504 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-21T00:38:36.504 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_1 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 673 Links: 1 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-21 00:38:11.521634282 +0000 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-21 00:35:36.392943720 +0000 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-21 00:35:36.392943720 +0000 2024-06-21T00:38:36.564 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-21 00:35:36.392943720 +0000 2024-06-21T00:38:36.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-21T00:38:36.632 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-21T00:38:36.632 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-21T00:38:36.632 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000265275 s, 1.9 MB/s 2024-06-21T00:38:36.633 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-21T00:38:36.693 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_2 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 693 Links: 1 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-21 00:38:11.521634282 +0000 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-21 00:35:36.731934717 +0000 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-21 00:35:36.731934717 +0000 2024-06-21T00:38:36.752 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-21 00:35:36.731934717 +0000 2024-06-21T00:38:36.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-21T00:38:36.818 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-21T00:38:36.818 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-21T00:38:36.818 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000237803 s, 2.2 MB/s 2024-06-21T00:38:36.820 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-21T00:38:36.879 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_3 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 713 Links: 1 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-21 00:38:11.521634282 +0000 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-21 00:35:37.080925449 +0000 2024-06-21T00:38:36.936 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-21 00:35:37.080925449 +0000 2024-06-21T00:38:36.937 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-21 00:35:37.080925449 +0000 2024-06-21T00:38:36.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-21T00:38:37.002 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-21T00:38:37.002 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-21T00:38:37.003 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000213062 s, 2.4 MB/s 2024-06-21T00:38:37.004 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-21T00:38:37.061 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_4 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi120=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: Saving service mgr spec with placement smithi027=y;smithi120=x;count:2 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:36 smithi120 ceph-mon[25975]: Deploying daemon mgr.x on smithi120 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 730 Links: 1 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-21 00:38:11.522634256 +0000 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-21 00:35:37.427916233 +0000 2024-06-21T00:38:37.119 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-21 00:35:37.427916233 +0000 2024-06-21T00:38:37.120 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-21 00:35:37.427916233 +0000 2024-06-21T00:38:37.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-21T00:38:37.185 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-21T00:38:37.185 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-21T00:38:37.185 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000214129 s, 2.4 MB/s 2024-06-21T00:38:37.186 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi120=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: Saving service mgr spec with placement smithi027=y;smithi120=x;count:2 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:37.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[25136]: Deploying daemon mgr.x on smithi120 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi120=x", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: Saving service mgr spec with placement smithi027=y;smithi120=x;count:2 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:37.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:36 smithi027 ceph-mon[29814]: Deploying daemon mgr.x on smithi120 2024-06-21T00:38:37.244 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:38:37.244 DEBUG:teuthology.orchestra.run.smithi120:> dd if=/scratch_devs of=/dev/stdout 2024-06-21T00:38:37.261 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-21T00:38:37.261 DEBUG:teuthology.orchestra.run.smithi120:> stat /dev/vg_nvme/lv_1 2024-06-21T00:38:37.316 INFO:teuthology.orchestra.run.smithi120.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-21T00:38:37.316 INFO:teuthology.orchestra.run.smithi120.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:37.316 INFO:teuthology.orchestra.run.smithi120.stdout:Device: 5h/5d Inode: 674 Links: 1 2024-06-21T00:38:37.316 INFO:teuthology.orchestra.run.smithi120.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:37.316 INFO:teuthology.orchestra.run.smithi120.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:37.317 INFO:teuthology.orchestra.run.smithi120.stdout:Access: 2024-06-21 00:38:20.842476165 +0000 2024-06-21T00:38:37.317 INFO:teuthology.orchestra.run.smithi120.stdout:Modify: 2024-06-21 00:35:36.350879021 +0000 2024-06-21T00:38:37.317 INFO:teuthology.orchestra.run.smithi120.stdout:Change: 2024-06-21 00:35:36.350879021 +0000 2024-06-21T00:38:37.317 INFO:teuthology.orchestra.run.smithi120.stdout: Birth: 2024-06-21 00:35:36.350879021 +0000 2024-06-21T00:38:37.317 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-21T00:38:37.380 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records in 2024-06-21T00:38:37.380 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records out 2024-06-21T00:38:37.380 INFO:teuthology.orchestra.run.smithi120.stderr:512 bytes copied, 0.000208271 s, 2.5 MB/s 2024-06-21T00:38:37.382 DEBUG:teuthology.orchestra.run.smithi120:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-21T00:38:37.438 DEBUG:teuthology.orchestra.run.smithi120:> stat /dev/vg_nvme/lv_2 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Device: 5h/5d Inode: 694 Links: 1 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Access: 2024-06-21 00:38:20.842476165 +0000 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Modify: 2024-06-21 00:35:36.686871553 +0000 2024-06-21T00:38:37.493 INFO:teuthology.orchestra.run.smithi120.stdout:Change: 2024-06-21 00:35:36.686871553 +0000 2024-06-21T00:38:37.494 INFO:teuthology.orchestra.run.smithi120.stdout: Birth: 2024-06-21 00:35:36.686871553 +0000 2024-06-21T00:38:37.494 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-21T00:38:37.556 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records in 2024-06-21T00:38:37.556 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records out 2024-06-21T00:38:37.556 INFO:teuthology.orchestra.run.smithi120.stderr:512 bytes copied, 0.000196938 s, 2.6 MB/s 2024-06-21T00:38:37.557 DEBUG:teuthology.orchestra.run.smithi120:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-21T00:38:37.614 DEBUG:teuthology.orchestra.run.smithi120:> stat /dev/vg_nvme/lv_3 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Device: 5h/5d Inode: 711 Links: 1 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Access: 2024-06-21 00:38:20.843476142 +0000 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Modify: 2024-06-21 00:35:37.026863996 +0000 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout:Change: 2024-06-21 00:35:37.026863996 +0000 2024-06-21T00:38:37.677 INFO:teuthology.orchestra.run.smithi120.stdout: Birth: 2024-06-21 00:35:37.026863996 +0000 2024-06-21T00:38:37.677 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-21T00:38:37.775 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records in 2024-06-21T00:38:37.775 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records out 2024-06-21T00:38:37.775 INFO:teuthology.orchestra.run.smithi120.stderr:512 bytes copied, 0.000179903 s, 2.8 MB/s 2024-06-21T00:38:37.776 DEBUG:teuthology.orchestra.run.smithi120:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-21T00:38:37.792 DEBUG:teuthology.orchestra.run.smithi120:> stat /dev/vg_nvme/lv_4 2024-06-21T00:38:37.852 INFO:teuthology.orchestra.run.smithi120.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-21T00:38:37.852 INFO:teuthology.orchestra.run.smithi120.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Device: 5h/5d Inode: 732 Links: 1 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Context: system_u:object_r:device_t:s0 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Access: 2024-06-21 00:38:37.735100022 +0000 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Modify: 2024-06-21 00:35:37.348856838 +0000 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout:Change: 2024-06-21 00:35:37.348856838 +0000 2024-06-21T00:38:37.853 INFO:teuthology.orchestra.run.smithi120.stdout: Birth: 2024-06-21 00:35:37.348856838 +0000 2024-06-21T00:38:37.853 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-21T00:38:37.946 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records in 2024-06-21T00:38:37.946 INFO:teuthology.orchestra.run.smithi120.stderr:1+0 records out 2024-06-21T00:38:37.947 INFO:teuthology.orchestra.run.smithi120.stderr:512 bytes copied, 0.000195276 s, 2.6 MB/s 2024-06-21T00:38:37.948 DEBUG:teuthology.orchestra.run.smithi120:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-21T00:38:37.963 INFO:tasks.cephadm:Deploying osd.0 on smithi027 with /dev/vg_nvme/lv_4... 2024-06-21T00:38:37.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-21T00:38:38.181 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:37 smithi120 systemd[1]: Starting Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:38:38.497 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 podman[27128]: 2024-06-21 00:38:38.086225552 +0000 UTC m=+0.017588944 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-21T00:38:38.498 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 podman[27128]: 2024-06-21 00:38:38.204255127 +0000 UTC m=+0.135618564 container create 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_BRANCH=HEAD, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754) 2024-06-21T00:38:38.498 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 podman[27128]: 2024-06-21 00:38:38.390417257 +0000 UTC m=+0.321780656 container init 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD) 2024-06-21T00:38:38.498 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 podman[27128]: 2024-06-21 00:38:38.39314544 +0000 UTC m=+0.324508840 container start 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream) 2024-06-21T00:38:38.498 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mgr[27147]: -- 172.21.15.120:0/2869803409 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f4eddf8340 con 0x55f4eddd0800 2024-06-21T00:38:38.498 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 bash[27128]: 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 2024-06-21T00:38:38.771 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 systemd[1]: Started Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:38:38.771 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:38.533+0000 7f1fa6d64000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:38:38.771 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:38.668+0000 7f1fa6d64000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:38:38.771 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:38.772+0000 7f1fa6d64000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:38:39.094 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mon[25975]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:39.094 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:39.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:39.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:39.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:39.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[25136]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:39.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:39.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:39.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[29814]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:39.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:39.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:39.095+0000 7f1fa6d64000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:38:39.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:39.181+0000 7f1fa6d64000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:38:39.961 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-21T00:38:39.961 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-21T00:38:39.961 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-21T00:38:39.961 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-21T00:38:39.961 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155512 s, 674 MB/s 2024-06-21T00:38:39.962 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-21T00:38:40.568 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:40.314+0000 7f1fa6d64000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:38:40.568 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:40.475+0000 7f1fa6d64000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:38:40.568 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:40.555+0000 7f1fa6d64000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:38:40.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_4 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: Reconfiguring mgr.y (unknown last config time)... 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[25136]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:40.657 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: Reconfiguring mgr.y (unknown last config time)... 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:40.658 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:40 smithi027 ceph-mon[29814]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T00:38:40.861 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:40.861 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.861 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:40.862 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: Reconfiguring mgr.y (unknown last config time)... 2024-06-21T00:38:40.862 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:38:40.862 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:38:40.862 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:40.862 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-mon[25975]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T00:38:41.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:40.862+0000 7f1fa6d64000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:38:41.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:40.951+0000 7f1fa6d64000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:38:41.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:41.042+0000 7f1fa6d64000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:38:41.571 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:41.278+0000 7f1fa6d64000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:38:41.608 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:38:41.608 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:38:41.608 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.608 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.608 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:41.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:41.710+0000 7f1fa6d64000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:38:41.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:38:41.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:38:41.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:42.571 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:42 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:42.190+0000 7f1fa6d64000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:38:42.572 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:42 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:42.282+0000 7f1fa6d64000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:38:42.572 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:42 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:42.571+0000 7f1fa6d64000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:38:42.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:42 smithi027 ceph-mon[25136]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:42.595 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:42 smithi027 ceph-mon[29814]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:42.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:42 smithi120 ceph-mon[25975]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:42.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:42 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:42.659+0000 7f1fa6d64000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:38:43.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:43.059+0000 7f1fa6d64000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:38:43.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:43.188+0000 7f1fa6d64000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:38:43.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:43.278+0000 7f1fa6d64000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:38:43.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[25136]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:43.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:38:43.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:38:43.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:43.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[29814]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:43.680 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:38:43.680 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:38:43.680 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:43 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:43.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:43.390+0000 7f1fa6d64000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:38:43.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:43.478+0000 7f1fa6d64000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:38:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-mon[25975]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:38:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:38:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:38:43.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:43 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:44.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:44.015+0000 7f1fa6d64000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:38:44.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:38:44.369+0000 7f1fa6d64000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:38:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T00:38:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:38:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:38:44.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:38:44.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:44 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:38:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:38:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:44 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1439274134' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:38:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: mgrmap e14: y(active, since 63s), standbys: x 2024-06-21T00:38:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:38:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3188430718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]': finished 2024-06-21T00:38:45.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: osdmap e5: 1 total, 0 up, 1 in 2024-06-21T00:38:45.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:45 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: mgrmap e14: y(active, since 63s), standbys: x 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3188430718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]': finished 2024-06-21T00:38:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: osdmap e5: 1 total, 0 up, 1 in 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: mgrmap e14: y(active, since 63s), standbys: x 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3188430718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]: dispatch 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d721abb8-7102-4cd5-8c27-42f36634e958"}]': finished 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: osdmap e5: 1 total, 0 up, 1 in 2024-06-21T00:38:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:45 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:38:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:46 smithi120 ceph-mon[25975]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:46 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4137082216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:38:46.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:46 smithi027 ceph-mon[25136]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:46.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:46 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4137082216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:38:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:46 smithi027 ceph-mon[29814]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:46 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4137082216' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:38:48.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:48 smithi120 ceph-mon[25975]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:48.912 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:48 smithi027 ceph-mon[25136]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:48.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:48 smithi027 ceph-mon[29814]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:50.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:50 smithi120 ceph-mon[25975]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:50.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:50 smithi027 ceph-mon[25136]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:50.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:50 smithi027 ceph-mon[29814]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:51.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:51 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T00:38:51.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:51 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:51.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:51 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T00:38:51.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:51 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:51.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:51 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T00:38:51.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:51 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:52.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:52 smithi120 ceph-mon[25975]: Deploying daemon osd.0 on smithi027 2024-06-21T00:38:52.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:52 smithi120 ceph-mon[25975]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:52.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:52 smithi027 ceph-mon[25136]: Deploying daemon osd.0 on smithi027 2024-06-21T00:38:52.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:52 smithi027 ceph-mon[25136]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:52.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:52 smithi027 ceph-mon[29814]: Deploying daemon osd.0 on smithi027 2024-06-21T00:38:52.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:52 smithi027 ceph-mon[29814]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:54.676 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:54 smithi027 ceph-mon[25136]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:54.677 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:54 smithi027 ceph-mon[29814]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:54 smithi120 ceph-mon[25975]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:56.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:56 smithi120 ceph-mon[25975]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:56.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:56 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:56.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:56 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:56.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:56 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:56.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:56 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:56.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[25136]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:56.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:56.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:56.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:56.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:56.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[29814]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:56.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:38:56.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:38:56.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:38:56.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:56 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:38:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:38:58 smithi120 ceph-mon[25975]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:58.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:38:58 smithi027 ceph-mon[25136]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:58.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:38:58 smithi027 ceph-mon[29814]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:38:59.633 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 0 on host 'smithi027' 2024-06-21T00:39:00.195 DEBUG:teuthology.orchestra.run.smithi027:osd.0> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.0.service 2024-06-21T00:39:00.238 INFO:tasks.cephadm:Deploying osd.1 on smithi027 with /dev/vg_nvme/lv_3... 2024-06-21T00:39:00.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[25136]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[25136]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[25136]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[29814]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:00.861 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[29814]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:00.862 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[29814]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:00.862 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:00 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:00 smithi120 ceph-mon[25975]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:00 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:00 smithi120 ceph-mon[25975]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:00 smithi120 ceph-mon[25975]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T00:39:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:00 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[25136]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[25136]: osdmap e6: 1 total, 0 up, 1 in 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[25136]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[25136]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:01.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[29814]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T00:39:01.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[29814]: osdmap e6: 1 total, 0 up, 1 in 2024-06-21T00:39:01.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:01.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[29814]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:01.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-mon[29814]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:01.904 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[32677]: 2024-06-21T00:39:01.644+0000 7ff989158700 -1 osd.0 0 waiting for initial osdmap 2024-06-21T00:39:01.904 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[32677]: 2024-06-21T00:39:01.646+0000 7ff9832ee700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:39:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:01 smithi120 ceph-mon[25975]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T00:39:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:01 smithi120 ceph-mon[25975]: osdmap e6: 1 total, 0 up, 1 in 2024-06-21T00:39:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:01 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:01 smithi120 ceph-mon[25975]: from='osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:01 smithi120 ceph-mon[25975]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:02.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:39:02.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:39:02.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:02.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:02.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: osdmap e7: 1 total, 0 up, 1 in 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: osdmap e7: 1 total, 0 up, 1 in 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:02.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:02 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: osdmap e7: 1 total, 0 up, 1 in 2024-06-21T00:39:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:03.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:02 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0122535 s, 856 MB/s 2024-06-21T00:39:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[25136]: osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173] boot 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[25136]: osdmap e8: 1 total, 1 up, 1 in 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[29814]: osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173] boot 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[29814]: osdmap e8: 1 total, 1 up, 1 in 2024-06-21T00:39:03.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:03 smithi120 ceph-mon[25975]: osd.0 [v2:172.21.15.27:6802/2514709173,v1:172.21.15.27:6803/2514709173] boot 2024-06-21T00:39:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:03 smithi120 ceph-mon[25975]: osdmap e8: 1 total, 1 up, 1 in 2024-06-21T00:39:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:39:04.889 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:04.889 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: osdmap e9: 1 total, 1 up, 1 in 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:04.890 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:04.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:04.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: osdmap e9: 1 total, 1 up, 1 in 2024-06-21T00:39:04.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:04.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: osdmap e9: 1 total, 1 up, 1 in 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:04.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:04.938 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:04.938 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:04.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_3 2024-06-21T00:39:06.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi027 2024-06-21T00:39:06.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi027 to 16064M 2024-06-21T00:39:06.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:06.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi027 2024-06-21T00:39:06.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi027 to 16064M 2024-06-21T00:39:06.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:05 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:05 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi027 2024-06-21T00:39:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:05 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi027 to 16064M 2024-06-21T00:39:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:05 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:07.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:06 smithi027 ceph-mon[25136]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:06 smithi027 ceph-mon[29814]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:07.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:06 smithi120 ceph-mon[25975]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[29814]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[25136]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:07 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:07 smithi120 ceph-mon[25975]: from='client.24142 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:07 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:07 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:07 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:09.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:08 smithi027 ceph-mon[25136]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:09.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:08 smithi027 ceph-mon[29814]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:09.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:08 smithi120 ceph-mon[25975]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:10.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/106421500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]': finished 2024-06-21T00:39:10.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: osdmap e10: 2 total, 1 up, 2 in 2024-06-21T00:39:10.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/130424163' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/106421500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]': finished 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: osdmap e10: 2 total, 1 up, 2 in 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:10.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:09 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/130424163' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/106421500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]: dispatch 2024-06-21T00:39:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7bff4817-8c21-4a34-87ff-36001678fd2b"}]': finished 2024-06-21T00:39:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: osdmap e10: 2 total, 1 up, 2 in 2024-06-21T00:39:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:10.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:09 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/130424163' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:11.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:10 smithi027 ceph-mon[25136]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:11.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:10 smithi027 ceph-mon[29814]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:11.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:10 smithi120 ceph-mon[25975]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:13.184 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:12 smithi027 ceph-mon[25136]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:13.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:12 smithi027 ceph-mon[29814]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:13.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:12 smithi120 ceph-mon[25975]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:15.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:14 smithi027 ceph-mon[29814]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:15.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:14 smithi027 ceph-mon[25136]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:14 smithi120 ceph-mon[25975]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:16.210 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:15 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T00:39:16.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:15 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:16.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:15 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T00:39:16.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:15 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:16.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:15 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T00:39:16.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:15 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:17.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:16 smithi027 ceph-mon[25136]: Deploying daemon osd.1 on smithi027 2024-06-21T00:39:17.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:16 smithi027 ceph-mon[25136]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:17.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:16 smithi027 ceph-mon[29814]: Deploying daemon osd.1 on smithi027 2024-06-21T00:39:17.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:16 smithi027 ceph-mon[29814]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:17.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:16 smithi120 ceph-mon[25975]: Deploying daemon osd.1 on smithi027 2024-06-21T00:39:17.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:16 smithi120 ceph-mon[25975]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:19.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:18 smithi027 ceph-mon[25136]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:19.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:18 smithi027 ceph-mon[29814]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:19.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:18 smithi120 ceph-mon[25975]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:20.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[25136]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:20.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:20.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:20.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:20.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:21.189 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:20 smithi120 ceph-mon[25975]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:21.189 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:21.189 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:21.189 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:21.189 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:21.218 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[29814]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:21.218 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:21.218 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:21.218 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:21.218 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:23.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:22 smithi027 ceph-mon[25136]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:23.195 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:22 smithi027 ceph-mon[29814]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:23.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:22 smithi120 ceph-mon[25975]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:23.714 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 1 on host 'smithi027' 2024-06-21T00:39:24.361 DEBUG:teuthology.orchestra.run.smithi027:osd.1> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.1.service 2024-06-21T00:39:24.363 INFO:tasks.cephadm:Deploying osd.2 on smithi027 with /dev/vg_nvme/lv_2... 2024-06-21T00:39:24.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-21T00:39:24.842 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[25136]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:24.842 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:24.842 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[25136]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[25136]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[29814]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[29814]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[29814]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:24.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:24 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:24 smithi120 ceph-mon[25975]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:24 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:24 smithi120 ceph-mon[25975]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:24 smithi120 ceph-mon[25975]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T00:39:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:24 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[25136]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T00:39:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[25136]: osdmap e11: 2 total, 1 up, 2 in 2024-06-21T00:39:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[25136]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[25136]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[29814]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[29814]: osdmap e11: 2 total, 1 up, 2 in 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[29814]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:25.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-mon[29814]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:25.981 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[35622]: 2024-06-21T00:39:25.730+0000 7f3e74cf4700 -1 osd.1 0 waiting for initial osdmap 2024-06-21T00:39:25.981 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:25 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[35622]: 2024-06-21T00:39:25.733+0000 7f3e7168f700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:39:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:25 smithi120 ceph-mon[25975]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T00:39:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:25 smithi120 ceph-mon[25975]: osdmap e11: 2 total, 1 up, 2 in 2024-06-21T00:39:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:25 smithi120 ceph-mon[25975]: from='osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:25 smithi120 ceph-mon[25975]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:26.706 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0145183 s, 722 MB/s 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 2024-06-21T00:39:26.707 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-21T00:39:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[25136]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[25136]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[25136]: osdmap e12: 2 total, 1 up, 2 in 2024-06-21T00:39:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[29814]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[29814]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[29814]: osdmap e12: 2 total, 1 up, 2 in 2024-06-21T00:39:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:26 smithi120 ceph-mon[25975]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-21T00:39:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:26 smithi120 ceph-mon[25975]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:26 smithi120 ceph-mon[25975]: osdmap e12: 2 total, 1 up, 2 in 2024-06-21T00:39:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:26 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:26 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[25136]: osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865] boot 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[25136]: osdmap e13: 2 total, 2 up, 2 in 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[29814]: osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865] boot 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[29814]: osdmap e13: 2 total, 2 up, 2 in 2024-06-21T00:39:27.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:27 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:27 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:39:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:27 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:39:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:27 smithi120 ceph-mon[25975]: osd.1 [v2:172.21.15.27:6810/521311865,v1:172.21.15.27:6811/521311865] boot 2024-06-21T00:39:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:27 smithi120 ceph-mon[25975]: osdmap e13: 2 total, 2 up, 2 in 2024-06-21T00:39:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:27 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:39:28.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_2 2024-06-21T00:39:28.830 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:28 smithi027 ceph-mon[29814]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:28.830 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:28 smithi027 ceph-mon[29814]: osdmap e14: 2 total, 2 up, 2 in 2024-06-21T00:39:28.831 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:28 smithi027 ceph-mon[25136]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:28.831 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:28 smithi027 ceph-mon[25136]: osdmap e14: 2 total, 2 up, 2 in 2024-06-21T00:39:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:28 smithi120 ceph-mon[25975]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:28 smithi120 ceph-mon[25975]: osdmap e14: 2 total, 2 up, 2 in 2024-06-21T00:39:30.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi027 2024-06-21T00:39:30.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi027 to 8032M 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi027 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi027 to 8032M 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:30.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi027 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi027 to 8032M 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:30.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:30.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[25136]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[29814]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:31.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:30 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:30 smithi120 ceph-mon[25975]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:30 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:30 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:30 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:32.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:31 smithi120 ceph-mon[25975]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:32.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:31 smithi027 ceph-mon[25136]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:32.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:31 smithi027 ceph-mon[29814]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[25136]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]: dispatch 2024-06-21T00:39:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]': finished 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[25136]: osdmap e15: 3 total, 2 up, 3 in 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[29814]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]: dispatch 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]': finished 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[29814]: osdmap e15: 3 total, 2 up, 3 in 2024-06-21T00:39:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:33.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:32 smithi120 ceph-mon[25975]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:33.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:32 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]: dispatch 2024-06-21T00:39:33.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:32 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/416324981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ad97f3ad-2a49-44a2-b30b-20f05146566a"}]': finished 2024-06-21T00:39:33.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:32 smithi120 ceph-mon[25975]: osdmap e15: 3 total, 2 up, 3 in 2024-06-21T00:39:33.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:33 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/311393039' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:33 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/311393039' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:34.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:33 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/311393039' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:34 smithi027 ceph-mon[25136]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:34 smithi027 ceph-mon[29814]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:34 smithi120 ceph-mon[25975]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:37.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:36 smithi027 ceph-mon[25136]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:37.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:36 smithi027 ceph-mon[29814]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:37.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:36 smithi120 ceph-mon[25975]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:39.168 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:38 smithi027 ceph-mon[25136]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:39.168 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:38 smithi027 ceph-mon[29814]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:39.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:38 smithi120 ceph-mon[25975]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:39 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T00:39:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:39 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:39 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T00:39:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:39 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:40.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:39 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T00:39:40.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:39 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:41.188 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:40 smithi027 ceph-mon[25136]: Deploying daemon osd.2 on smithi027 2024-06-21T00:39:41.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:40 smithi027 ceph-mon[25136]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:41.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:40 smithi027 ceph-mon[29814]: Deploying daemon osd.2 on smithi027 2024-06-21T00:39:41.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:40 smithi027 ceph-mon[29814]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:41.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:40 smithi120 ceph-mon[25975]: Deploying daemon osd.2 on smithi027 2024-06-21T00:39:41.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:40 smithi120 ceph-mon[25975]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:42.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:39:42.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:39:42.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:39:42.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:39:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:39:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:39:43.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:42 smithi027 ceph-mon[25136]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:43.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:42 smithi027 ceph-mon[29814]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:43.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:42 smithi120 ceph-mon[25975]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:45.024 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[29814]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[25136]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:45.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:44 smithi120 ceph-mon[25975]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:47.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:46 smithi027 ceph-mon[29814]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:47.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:46 smithi027 ceph-mon[25136]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:47.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:46 smithi120 ceph-mon[25975]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:47.871 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 2 on host 'smithi027' 2024-06-21T00:39:48.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:48.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:48.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:48.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T00:39:48.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:48.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:39:48.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:48.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:39:48.448 DEBUG:teuthology.orchestra.run.smithi027:osd.2> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.2.service 2024-06-21T00:39:48.450 INFO:tasks.cephadm:Deploying osd.3 on smithi027 with /dev/vg_nvme/lv_1... 2024-06-21T00:39:48.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-21T00:39:49.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:49 smithi120 ceph-mon[25975]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:49.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:49 smithi120 ceph-mon[25975]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T00:39:49.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:49 smithi120 ceph-mon[25975]: osdmap e16: 3 total, 2 up, 3 in 2024-06-21T00:39:49.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:49 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:49.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:49 smithi120 ceph-mon[25975]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[25136]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[25136]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T00:39:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[25136]: osdmap e16: 3 total, 2 up, 3 in 2024-06-21T00:39:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[25136]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[29814]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[29814]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[29814]: osdmap e16: 3 total, 2 up, 3 in 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:49.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-mon[29814]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:39:49.480 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[38526]: 2024-06-21T00:39:49.104+0000 7fc13a979700 -1 osd.2 0 waiting for initial osdmap 2024-06-21T00:39:49.480 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:49 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[38526]: 2024-06-21T00:39:49.108+0000 7fc135310700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:39:50.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[25136]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:50.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[25136]: osdmap e17: 3 total, 2 up, 3 in 2024-06-21T00:39:50.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:50.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:50.364 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[29814]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:50.364 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[29814]: osdmap e17: 3 total, 2 up, 3 in 2024-06-21T00:39:50.364 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:50.364 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:50 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:50 smithi120 ceph-mon[25975]: from='osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:39:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:50 smithi120 ceph-mon[25975]: osdmap e17: 3 total, 2 up, 3 in 2024-06-21T00:39:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:50 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:50 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:51.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:39:51.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:39:51.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:51.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934] boot 2024-06-21T00:39:51.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: osdmap e18: 3 total, 3 up, 3 in 2024-06-21T00:39:51.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:51 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:51.410 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:39:51.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:39:51.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:51.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934] boot 2024-06-21T00:39:51.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: osdmap e18: 3 total, 3 up, 3 in 2024-06-21T00:39:51.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:51.412 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:39:51.412 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:39:51.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-21T00:39:51.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: osd.2 [v2:172.21.15.27:6818/2017021934,v1:172.21.15.27:6819/2017021934] boot 2024-06-21T00:39:51.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: osdmap e18: 3 total, 3 up, 3 in 2024-06-21T00:39:51.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:51 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:39:52.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: osdmap e19: 3 total, 3 up, 3 in 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:52 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-21T00:39:52.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: osdmap e19: 3 total, 3 up, 3 in 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: osdmap e19: 3 total, 3 up, 3 in 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:39:52.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:52.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:52 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.016458 s, 637 MB/s 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 2024-06-21T00:39:52.602 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-21T00:39:53.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi027 2024-06-21T00:39:53.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi027 to 5354M 2024-06-21T00:39:53.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:53.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-21T00:39:53.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: osdmap e20: 3 total, 3 up, 3 in 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi027 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi027 to 5354M 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: osdmap e20: 3 total, 3 up, 3 in 2024-06-21T00:39:53.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:53 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-21T00:39:53.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_1 2024-06-21T00:39:53.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi027 2024-06-21T00:39:53.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi027 to 5354M 2024-06-21T00:39:53.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:53.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-21T00:39:53.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: osdmap e20: 3 total, 3 up, 3 in 2024-06-21T00:39:53.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:53 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-21T00:39:54.227 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:53 smithi027 sudo[39823]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:54.227 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:53 smithi027 sudo[39823]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:54.227 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39823]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[29814]: osdmap e21: 3 total, 3 up, 3 in 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[29814]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[25136]: osdmap e21: 3 total, 3 up, 3 in 2024-06-21T00:39:54.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:54 smithi027 ceph-mon[25136]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:54.480 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39829]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:54.480 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39829]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:54.480 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39829]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:54 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-21T00:39:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:54 smithi120 ceph-mon[25975]: osdmap e21: 3 total, 3 up, 3 in 2024-06-21T00:39:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:54 smithi120 ceph-mon[25975]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:54.988 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39857]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:54.988 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39857]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:54.988 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:54 smithi027 sudo[39857]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:55.575 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39917]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:55.575 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39917]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:55.575 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39917]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:55.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[25136]: osdmap e22: 3 total, 3 up, 3 in 2024-06-21T00:39:55.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[29814]: osdmap e22: 3 total, 3 up, 3 in 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:55.576 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:55 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:55 smithi120 ceph-mon[25975]: osdmap e22: 3 total, 3 up, 3 in 2024-06-21T00:39:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:55 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:55 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:39:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:55 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:39:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:55 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:39:56.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39964]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:56.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39964]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:56.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:55 smithi027 sudo[39964]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:56.595 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[39983]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:56.596 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[39983]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:56.596 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[39983]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:56.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[40005]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:56.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[40005]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:56.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:56 smithi027 sudo[40005]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:57.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[29814]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:57.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[29814]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-21T00:39:57.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[29814]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40018]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40018]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40018]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[25136]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[25136]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[25136]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:57 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:57 smithi120 ceph-mon[25975]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:39:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:57 smithi120 ceph-mon[25975]: mgrmap e15: y(active, since 2m), standbys: x 2024-06-21T00:39:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:57 smithi120 ceph-mon[25975]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:57 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40052]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40052]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:57 smithi027 sudo[40052]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:58.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 sudo[40055]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:58.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 sudo[40055]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:58.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 sudo[40055]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:58.590 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27483]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-21T00:39:58.590 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27483]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:58.590 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27483]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27486]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-21T00:39:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27486]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-21T00:39:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 sudo[27486]: pam_unix(sudo:session): session closed for user root 2024-06-21T00:39:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]': finished 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: osdmap e23: 4 total, 3 up, 4 in 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:39:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]': finished 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: osdmap e23: 4 total, 3 up, 4 in 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]: dispatch 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/64067298' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "538dc8d0-a030-47be-9c0b-7006e0fb6708"}]': finished 2024-06-21T00:39:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: osdmap e23: 4 total, 3 up, 4 in 2024-06-21T00:39:58.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:39:58.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:39:58.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:39:58.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:39:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:59 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:59 smithi120 ceph-mon[25975]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:59 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:39:59 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2180147094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[25136]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2180147094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-21T00:39:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[29814]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-21T00:39:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:39:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:39:59 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2180147094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:00.938 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:00 smithi027 ceph-mon[29814]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:00.938 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:00 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T00:40:00.938 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:00 smithi027 ceph-mon[25136]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:00.938 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:00 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T00:40:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:00 smithi120 ceph-mon[25975]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:01.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:00 smithi120 ceph-mon[25975]: overall HEALTH_OK 2024-06-21T00:40:02.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:02 smithi027 ceph-mon[25136]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:02.970 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:02 smithi027 ceph-mon[29814]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:02 smithi120 ceph-mon[25975]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:04.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[25136]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-06-21T00:40:04.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T00:40:04.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:04.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[29814]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-06-21T00:40:04.922 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T00:40:04.922 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:04 smithi120 ceph-mon[25975]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-06-21T00:40:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T00:40:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:05.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:05 smithi027 ceph-mon[25136]: Deploying daemon osd.3 on smithi027 2024-06-21T00:40:05.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:05 smithi027 ceph-mon[29814]: Deploying daemon osd.3 on smithi027 2024-06-21T00:40:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:05 smithi120 ceph-mon[25975]: Deploying daemon osd.3 on smithi027 2024-06-21T00:40:06.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:06 smithi027 ceph-mon[25136]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-06-21T00:40:06.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:06 smithi027 ceph-mon[29814]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-06-21T00:40:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:06 smithi120 ceph-mon[25975]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-06-21T00:40:08.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:08 smithi027 ceph-mon[25136]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-06-21T00:40:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:08 smithi027 ceph-mon[29814]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-06-21T00:40:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:08 smithi120 ceph-mon[25975]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-06-21T00:40:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[25136]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-06-21T00:40:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[29814]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-06-21T00:40:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:10 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:11.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:10 smithi120 ceph-mon[25975]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 69 KiB/s wr, 4 op/s 2024-06-21T00:40:11.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:10 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:11.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:10 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:11.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:10 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:11.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:10 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:12.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:12 smithi027 ceph-mon[25136]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:12.888 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:12 smithi027 ceph-mon[29814]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:12.898 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 3 on host 'smithi027' 2024-06-21T00:40:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:12 smithi120 ceph-mon[25975]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:13.594 DEBUG:teuthology.orchestra.run.smithi027:osd.3> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.3.service 2024-06-21T00:40:13.596 INFO:tasks.cephadm:Deploying osd.4 on smithi120 with /dev/vg_nvme/lv_4... 2024-06-21T00:40:13.596 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-21T00:40:13.908 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:13 smithi120 ceph-mon[25975]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.908 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:13 smithi120 ceph-mon[25975]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.908 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:13 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[25136]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[25136]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[29814]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[29814]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T00:40:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:13 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:14.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:14.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T00:40:14.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: osdmap e24: 4 total, 3 up, 4 in 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: osdmap e24: 4 total, 3 up, 4 in 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:14.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:14.981 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[41495]: 2024-06-21T00:40:14.709+0000 7f14084b6700 -1 osd.3 0 waiting for initial osdmap 2024-06-21T00:40:14.981 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 00:40:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[41495]: 2024-06-21T00:40:14.713+0000 7f1402e4d700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:40:15.054 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-21T00:40:15.054 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T00:40:15.054 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: osdmap e24: 4 total, 3 up, 4 in 2024-06-21T00:40:15.054 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:15.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: from='osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:15.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T00:40:15.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:14 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:15.536 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-21T00:40:15.537 INFO:teuthology.orchestra.run.smithi120.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-21T00:40:15.537 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10+0 records in 2024-06-21T00:40:15.537 INFO:teuthology.orchestra.run.smithi120.stderr:10+0 records out 2024-06-21T00:40:15.537 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130996 s, 800 MB/s 2024-06-21T00:40:15.537 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping successful for: 2024-06-21T00:40:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[25136]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:40:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[25136]: osdmap e25: 4 total, 3 up, 4 in 2024-06-21T00:40:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[29814]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:40:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[29814]: osdmap e25: 4 total, 3 up, 4 in 2024-06-21T00:40:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:15 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:15 smithi120 ceph-mon[25975]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-21T00:40:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:15 smithi120 ceph-mon[25975]: osdmap e25: 4 total, 3 up, 4 in 2024-06-21T00:40:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:15 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:15 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:16.154 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi120:vg_nvme/lv_4 2024-06-21T00:40:16.926 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:40:16.926 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199] boot 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: osdmap e26: 4 total, 4 up, 4 in 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199] boot 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: osdmap e26: 4 total, 4 up, 4 in 2024-06-21T00:40:16.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:16 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:40:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:40:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-21T00:40:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: osd.3 [v2:172.21.15.27:6826/3447780199,v1:172.21.15.27:6827/3447780199] boot 2024-06-21T00:40:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: osdmap e26: 4 total, 4 up, 4 in 2024-06-21T00:40:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:16 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:40:17.795 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: osdmap e27: 4 total, 4 up, 4 in 2024-06-21T00:40:17.795 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi027 2024-06-21T00:40:17.795 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi027 to 4016M 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:17.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: osdmap e27: 4 total, 4 up, 4 in 2024-06-21T00:40:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi027 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi027 to 4016M 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: osdmap e27: 4 total, 4 up, 4 in 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi027 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi027 to 4016M 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:17.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:17.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:19.244 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:18 smithi120 ceph-mon[25975]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:19.245 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:18 smithi120 ceph-mon[25975]: from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:19.245 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:18 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:19.245 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:18 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:19.245 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:18 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:19.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[25136]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[25136]: from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[29814]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[29814]: from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:19.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:18 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/2842306232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]': finished 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: osdmap e28: 5 total, 4 up, 5 in 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:21.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/620776747' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:21.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:21.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/2842306232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]': finished 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: osdmap e28: 5 total, 4 up, 5 in 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/620776747' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/2842306232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09479a17-ed8f-4454-871d-7288f3c60f8f"}]': finished 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: osdmap e28: 5 total, 4 up, 5 in 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:21.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/620776747' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:23.319 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:22 smithi120 ceph-mon[25975]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:23.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:22 smithi027 ceph-mon[25136]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:23.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:22 smithi027 ceph-mon[29814]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:24 smithi120 ceph-mon[25975]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:24 smithi027 ceph-mon[25136]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:25.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:24 smithi027 ceph-mon[29814]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:27 smithi120 ceph-mon[25975]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:27 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T00:40:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:27 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:27 smithi120 ceph-mon[25975]: Deploying daemon osd.4 on smithi120 2024-06-21T00:40:27.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[25136]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:27.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T00:40:27.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:27.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[25136]: Deploying daemon osd.4 on smithi120 2024-06-21T00:40:27.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[29814]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:27.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T00:40:27.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:27.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:26 smithi027 ceph-mon[29814]: Deploying daemon osd.4 on smithi120 2024-06-21T00:40:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:29 smithi120 ceph-mon[25975]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:29.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:29 smithi027 ceph-mon[25136]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:29.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:29 smithi027 ceph-mon[29814]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:31.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:31 smithi027 ceph-mon[25136]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:31.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:31 smithi027 ceph-mon[29814]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:31.360 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:31 smithi120 ceph-mon[25975]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:32.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:32 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:32.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:32.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:32.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:32.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:32.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:32.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:32.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:32.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:32 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:33.257 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:33 smithi120 ceph-mon[25975]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:33.257 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:33.257 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:33 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:33.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[29814]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:33.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:33.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:33.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[25136]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:33.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:33.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:33 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:34.156 INFO:teuthology.orchestra.run.smithi120.stdout:Created osd(s) 4 on host 'smithi120' 2024-06-21T00:40:34.848 DEBUG:teuthology.orchestra.run.smithi120:osd.4> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.4.service 2024-06-21T00:40:34.850 INFO:tasks.cephadm:Deploying osd.5 on smithi120 with /dev/vg_nvme/lv_3... 2024-06-21T00:40:34.850 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-21T00:40:35.100 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:35.100 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:35.100 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:35.100 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:35.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:35.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:35.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:36.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:36.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:35 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:36.459 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T00:40:36.459 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: osdmap e29: 5 total, 4 up, 5 in 2024-06-21T00:40:36.459 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:36.459 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.459 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.460 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-mon[25975]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:36.460 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[29386]: 2024-06-21T00:40:36.166+0000 7f3cbff87700 -1 osd.4 0 waiting for initial osdmap 2024-06-21T00:40:36.460 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 00:40:36 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[29386]: 2024-06-21T00:40:36.170+0000 7f3cba11d700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:40:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T00:40:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: osdmap e29: 5 total, 4 up, 5 in 2024-06-21T00:40:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[25136]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: osdmap e29: 5 total, 4 up, 5 in 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: from='osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:36.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:36 smithi027 ceph-mon[29814]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:37.348 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-21T00:40:37.348 INFO:teuthology.orchestra.run.smithi120.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-21T00:40:37.348 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10+0 records in 2024-06-21T00:40:37.348 INFO:teuthology.orchestra.run.smithi120.stderr:10+0 records out 2024-06-21T00:40:37.348 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128805 s, 814 MB/s 2024-06-21T00:40:37.349 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping successful for: 2024-06-21T00:40:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[25136]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:40:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[25136]: osdmap e30: 5 total, 4 up, 5 in 2024-06-21T00:40:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:37.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[29814]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:40:37.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[29814]: osdmap e30: 5 total, 4 up, 5 in 2024-06-21T00:40:37.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:37.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:37 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:37.593 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:37 smithi120 ceph-mon[25975]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:40:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:37 smithi120 ceph-mon[25975]: osdmap e30: 5 total, 4 up, 5 in 2024-06-21T00:40:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:37 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:37 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.110 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi120:vg_nvme/lv_3 2024-06-21T00:40:38.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:40:38.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:40:38.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894] boot 2024-06-21T00:40:38.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: osdmap e31: 5 total, 5 up, 5 in 2024-06-21T00:40:38.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:38 smithi120 ceph-mon[25975]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894] boot 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: osdmap e31: 5 total, 5 up, 5 in 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[29814]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: osd.4 [v2:172.21.15.120:6800/4213715894,v1:172.21.15.120:6801/4213715894] boot 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: osdmap e31: 5 total, 5 up, 5 in 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:40:38.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:38 smithi027 ceph-mon[25136]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-21T00:40:39.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:39 smithi120 ceph-mon[25975]: osdmap e32: 5 total, 5 up, 5 in 2024-06-21T00:40:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:39 smithi027 ceph-mon[25136]: osdmap e32: 5 total, 5 up, 5 in 2024-06-21T00:40:39.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:39 smithi027 ceph-mon[29814]: osdmap e32: 5 total, 5 up, 5 in 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: osdmap e33: 5 total, 5 up, 5 in 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi120 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:40.203 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi120 to 17088M 2024-06-21T00:40:40.204 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.204 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:40 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: osdmap e33: 5 total, 5 up, 5 in 2024-06-21T00:40:40.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:40.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi120 2024-06-21T00:40:40.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi120 to 17088M 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: osdmap e33: 5 total, 5 up, 5 in 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi120 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi120 to 17088M 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:40.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:40 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:41.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:41 smithi120 ceph-mon[25975]: osdmap e34: 5 total, 5 up, 5 in 2024-06-21T00:40:41.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:41 smithi120 ceph-mon[25975]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:41.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:41.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:41.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[25136]: osdmap e34: 5 total, 5 up, 5 in 2024-06-21T00:40:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[25136]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[29814]: osdmap e34: 5 total, 5 up, 5 in 2024-06-21T00:40:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[29814]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:40:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:40:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:40:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:42.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:42 smithi120 ceph-mon[25975]: osdmap e35: 5 total, 5 up, 5 in 2024-06-21T00:40:42.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:42 smithi120 ceph-mon[25975]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:42.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:42 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:40:42.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:42 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:40:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[29814]: osdmap e35: 5 total, 5 up, 5 in 2024-06-21T00:40:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[29814]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:40:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:40:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[25136]: osdmap e35: 5 total, 5 up, 5 in 2024-06-21T00:40:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[25136]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:40:42.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:40:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/923669151' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]': finished 2024-06-21T00:40:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[29814]: osdmap e36: 6 total, 5 up, 6 in 2024-06-21T00:40:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:43.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/923669151' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]': finished 2024-06-21T00:40:43.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[25136]: osdmap e36: 6 total, 5 up, 6 in 2024-06-21T00:40:43.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:43 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:43.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:43 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/923669151' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:43 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]: dispatch 2024-06-21T00:40:43.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:43 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f10e337-c99d-4759-8ea7-ed600b3ad8e4"}]': finished 2024-06-21T00:40:43.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:43 smithi120 ceph-mon[25975]: osdmap e36: 6 total, 5 up, 6 in 2024-06-21T00:40:43.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:43 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:44.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:44 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/2701090511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:44.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:44 smithi027 ceph-mon[29814]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-06-21T00:40:44.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:44 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/2701090511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:44.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:44 smithi027 ceph-mon[25136]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-06-21T00:40:44.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:44 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/2701090511' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:40:44.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:44 smithi120 ceph-mon[25975]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-06-21T00:40:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:46 smithi120 ceph-mon[25975]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:40:46.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:46 smithi027 ceph-mon[25136]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:40:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:46 smithi027 ceph-mon[29814]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:40:48.803 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:48 smithi120 ceph-mon[25975]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-06-21T00:40:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:48 smithi027 ceph-mon[25136]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-06-21T00:40:48.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:48 smithi027 ceph-mon[29814]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-06-21T00:40:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:49 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T00:40:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:49 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:49 smithi120 ceph-mon[25975]: Deploying daemon osd.5 on smithi120 2024-06-21T00:40:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T00:40:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[25136]: Deploying daemon osd.5 on smithi120 2024-06-21T00:40:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T00:40:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:49 smithi027 ceph-mon[29814]: Deploying daemon osd.5 on smithi120 2024-06-21T00:40:50.758 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:50 smithi120 ceph-mon[25975]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-06-21T00:40:50.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:50 smithi027 ceph-mon[25136]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-06-21T00:40:50.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:50 smithi027 ceph-mon[29814]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-06-21T00:40:52.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:52 smithi120 ceph-mon[25975]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:40:52.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:52 smithi027 ceph-mon[29814]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:40:52.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:52 smithi027 ceph-mon[25136]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:40:54.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[29814]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:54.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:54.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:54.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[25136]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:54 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:55.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:54 smithi120 ceph-mon[25975]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:55.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:54 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:55.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:54 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:40:55.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:54 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:40:55.011 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:54 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:40:56.730 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:56 smithi120 ceph-mon[25975]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:56.760 INFO:teuthology.orchestra.run.smithi120.stdout:Created osd(s) 5 on host 'smithi120' 2024-06-21T00:40:56.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:56 smithi027 ceph-mon[25136]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:56.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:56 smithi027 ceph-mon[29814]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:57.334 DEBUG:teuthology.orchestra.run.smithi120:osd.5> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.5.service 2024-06-21T00:40:57.337 INFO:tasks.cephadm:Deploying osd.6 on smithi120 with /dev/vg_nvme/lv_2... 2024-06-21T00:40:57.337 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-21T00:40:58.012 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:57 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.012 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:57 smithi120 ceph-mon[25975]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.012 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:57 smithi120 ceph-mon[25975]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.013 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:57 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[25136]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[25136]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[29814]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[29814]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T00:40:58.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:57 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:40:58.894 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:58.894 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T00:40:58.894 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: osdmap e37: 6 total, 5 up, 6 in 2024-06-21T00:40:58.894 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:58.894 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:58.895 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-mon[25975]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:58.895 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[32250]: 2024-06-21T00:40:58.786+0000 7fd3303ff700 -1 osd.5 0 waiting for initial osdmap 2024-06-21T00:40:58.895 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 00:40:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[32250]: 2024-06-21T00:40:58.790+0000 7fd328d92700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:40:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T00:40:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: osdmap e37: 6 total, 5 up, 6 in 2024-06-21T00:40:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[25136]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: osdmap e37: 6 total, 5 up, 6 in 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: from='osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:58 smithi027 ceph-mon[29814]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:40:59.511 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-21T00:40:59.512 INFO:teuthology.orchestra.run.smithi120.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-21T00:40:59.512 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10+0 records in 2024-06-21T00:40:59.512 INFO:teuthology.orchestra.run.smithi120.stderr:10+0 records out 2024-06-21T00:40:59.512 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129149 s, 812 MB/s 2024-06-21T00:40:59.512 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping successful for: 2024-06-21T00:41:00.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:00.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: osdmap e38: 6 total, 5 up, 6 in 2024-06-21T00:41:00.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105] boot 2024-06-21T00:41:00.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: osdmap e39: 6 total, 6 up, 6 in 2024-06-21T00:41:00.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:40:59 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: osdmap e38: 6 total, 5 up, 6 in 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105] boot 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: osdmap e39: 6 total, 6 up, 6 in 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: osdmap e38: 6 total, 5 up, 6 in 2024-06-21T00:41:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: osd.5 [v2:172.21.15.120:6808/1314144105,v1:172.21.15.120:6809/1314144105] boot 2024-06-21T00:41:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: osdmap e39: 6 total, 6 up, 6 in 2024-06-21T00:41:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:40:59 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:41:00.385 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi120:vg_nvme/lv_2 2024-06-21T00:41:01.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:00 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:41:01.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:00 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:41:01.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:00 smithi120 ceph-mon[25975]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:41:01.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:00 smithi120 ceph-mon[25975]: osdmap e40: 6 total, 6 up, 6 in 2024-06-21T00:41:01.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:41:01.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:41:01.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[25136]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:41:01.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[25136]: osdmap e40: 6 total, 6 up, 6 in 2024-06-21T00:41:01.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:41:01.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:41:01.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[29814]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-21T00:41:01.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:00 smithi027 ceph-mon[29814]: osdmap e40: 6 total, 6 up, 6 in 2024-06-21T00:41:02.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:02 smithi120 ceph-mon[25975]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:02.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:02 smithi120 ceph-mon[25975]: osdmap e41: 6 total, 6 up, 6 in 2024-06-21T00:41:02.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:02 smithi027 ceph-mon[25136]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:02.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:02 smithi027 ceph-mon[25136]: osdmap e41: 6 total, 6 up, 6 in 2024-06-21T00:41:02.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:02 smithi027 ceph-mon[29814]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:02.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:02 smithi027 ceph-mon[29814]: osdmap e41: 6 total, 6 up, 6 in 2024-06-21T00:41:03.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: osdmap e42: 6 total, 6 up, 6 in 2024-06-21T00:41:03.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi120 2024-06-21T00:41:03.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi120 to 8544M 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:03.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:03 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: osdmap e42: 6 total, 6 up, 6 in 2024-06-21T00:41:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi120 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi120 to 8544M 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: osdmap e42: 6 total, 6 up, 6 in 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi120 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi120 to 8544M 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:03.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:03 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: osdmap e43: 6 total, 6 up, 6 in 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:04.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:04 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: osdmap e43: 6 total, 6 up, 6 in 2024-06-21T00:41:04.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: osdmap e43: 6 total, 6 up, 6 in 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:04.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:04 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/496225603' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]': finished 2024-06-21T00:41:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: osdmap e44: 7 total, 6 up, 7 in 2024-06-21T00:41:06.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:06.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:06 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/1150704572' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/496225603' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]': finished 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: osdmap e44: 7 total, 6 up, 7 in 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/1150704572' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/496225603' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]: dispatch 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9728d5f9-851a-4109-b5de-780d4ddc63b9"}]': finished 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: osdmap e44: 7 total, 6 up, 7 in 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:06.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:06 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/1150704572' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:08.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:08 smithi120 ceph-mon[25975]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:08 smithi027 ceph-mon[25136]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:08.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:08 smithi027 ceph-mon[29814]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:10.819 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:10 smithi120 ceph-mon[25975]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:10 smithi027 ceph-mon[25136]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:10 smithi027 ceph-mon[29814]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:12.846 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:12 smithi120 ceph-mon[25975]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:12.846 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:12 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T00:41:12.846 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:12 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:12.846 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:12 smithi120 ceph-mon[25975]: Deploying daemon osd.6 on smithi120 2024-06-21T00:41:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[25136]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T00:41:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[25136]: Deploying daemon osd.6 on smithi120 2024-06-21T00:41:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[29814]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T00:41:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:12 smithi027 ceph-mon[29814]: Deploying daemon osd.6 on smithi120 2024-06-21T00:41:14.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:14 smithi120 ceph-mon[25975]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:14.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:14 smithi027 ceph-mon[29814]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:14.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:14 smithi027 ceph-mon[25136]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:16.723 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:16 smithi120 ceph-mon[25975]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:16 smithi027 ceph-mon[25136]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:16 smithi027 ceph-mon[29814]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:18.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:18.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:18.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:18.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:17 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:18.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:18.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:18.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:18.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:17 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:18 smithi120 ceph-mon[25975]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:19.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:18 smithi027 ceph-mon[25136]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:19.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:18 smithi027 ceph-mon[29814]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:19.795 INFO:teuthology.orchestra.run.smithi120.stdout:Created osd(s) 6 on host 'smithi120' 2024-06-21T00:41:20.509 DEBUG:teuthology.orchestra.run.smithi120:osd.6> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.6.service 2024-06-21T00:41:20.512 INFO:tasks.cephadm:Deploying osd.7 on smithi120 with /dev/vg_nvme/lv_1... 2024-06-21T00:41:20.512 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-21T00:41:21.049 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[25136]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[25136]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[25136]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[29814]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[29814]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[29814]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:20 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:21.063 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:20 smithi120 ceph-mon[25975]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:21.063 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:21.063 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:20 smithi120 ceph-mon[25975]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.063 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:20 smithi120 ceph-mon[25975]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T00:41:21.063 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:20 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[25136]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T00:41:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[25136]: osdmap e45: 7 total, 6 up, 7 in 2024-06-21T00:41:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[25136]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[25136]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[29814]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T00:41:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[29814]: osdmap e45: 7 total, 6 up, 7 in 2024-06-21T00:41:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[29814]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:21 smithi027 ceph-mon[29814]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-mon[25975]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T00:41:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-mon[25975]: osdmap e45: 7 total, 6 up, 7 in 2024-06-21T00:41:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-mon[25975]: from='osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-mon[25975]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:22.368 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[35130]: 2024-06-21T00:41:21.960+0000 7fde7ad72700 -1 osd.6 0 waiting for initial osdmap 2024-06-21T00:41:22.368 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 00:41:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[35130]: 2024-06-21T00:41:21.967+0000 7fde75f0a700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:41:22.504 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-21T00:41:22.504 INFO:teuthology.orchestra.run.smithi120.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-21T00:41:22.504 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10+0 records in 2024-06-21T00:41:22.505 INFO:teuthology.orchestra.run.smithi120.stderr:10+0 records out 2024-06-21T00:41:22.505 INFO:teuthology.orchestra.run.smithi120.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122468 s, 856 MB/s 2024-06-21T00:41:22.505 INFO:teuthology.orchestra.run.smithi120.stderr:--> Zapping successful for: 2024-06-21T00:41:23.165 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch daemon add osd smithi120:vg_nvme/lv_1 2024-06-21T00:41:23.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[25136]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[25136]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[25136]: osdmap e46: 7 total, 6 up, 7 in 2024-06-21T00:41:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[29814]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[29814]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[29814]: osdmap e46: 7 total, 6 up, 7 in 2024-06-21T00:41:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:22 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:23.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:22 smithi120 ceph-mon[25975]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:23.268 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:22 smithi120 ceph-mon[25975]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:23.268 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:22 smithi120 ceph-mon[25975]: osdmap e46: 7 total, 6 up, 7 in 2024-06-21T00:41:23.268 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:22 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:23.268 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:22 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:24.131 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:23 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:41:24.131 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:23 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:41:24.131 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:23 smithi120 ceph-mon[25975]: osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121] boot 2024-06-21T00:41:24.131 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:23 smithi120 ceph-mon[25975]: osdmap e47: 7 total, 7 up, 7 in 2024-06-21T00:41:24.131 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:23 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:41:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:41:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[25136]: osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121] boot 2024-06-21T00:41:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[25136]: osdmap e47: 7 total, 7 up, 7 in 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[29814]: osd.6 [v2:172.21.15.120:6816/766528121,v1:172.21.15.120:6817/766528121] boot 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[29814]: osdmap e47: 7 total, 7 up, 7 in 2024-06-21T00:41:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:23 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:41:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:24 smithi120 ceph-mon[25975]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:24 smithi120 ceph-mon[25975]: osdmap e48: 7 total, 7 up, 7 in 2024-06-21T00:41:25.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:24 smithi027 ceph-mon[25136]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:25.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:24 smithi027 ceph-mon[25136]: osdmap e48: 7 total, 7 up, 7 in 2024-06-21T00:41:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:24 smithi027 ceph-mon[29814]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-21T00:41:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:24 smithi027 ceph-mon[29814]: osdmap e48: 7 total, 7 up, 7 in 2024-06-21T00:41:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: osdmap e49: 7 total, 7 up, 7 in 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: osdmap e50: 7 total, 7 up, 7 in 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:26.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:25 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: osdmap e49: 7 total, 7 up, 7 in 2024-06-21T00:41:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: osdmap e50: 7 total, 7 up, 7 in 2024-06-21T00:41:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: osdmap e49: 7 total, 7 up, 7 in 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: osdmap e50: 7 total, 7 up, 7 in 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:26.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:25 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi120 2024-06-21T00:41:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi120 to 5696M 2024-06-21T00:41:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: osdmap e51: 7 total, 7 up, 7 in 2024-06-21T00:41:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi120 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi120 to 5696M 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: osdmap e51: 7 total, 7 up, 7 in 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:27.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:26 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:27.297 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi120 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi120 to 5696M 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: osdmap e51: 7 total, 7 up, 7 in 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-21T00:41:27.298 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:26 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:28.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:27 smithi120 ceph-mon[25975]: from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:28.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:27 smithi027 ceph-mon[25136]: from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:28.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:27 smithi027 ceph-mon[29814]: from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi120:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:28 smithi120 ceph-mon[25975]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:29.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:28 smithi027 ceph-mon[25136]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:29.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:28 smithi027 ceph-mon[29814]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/1545332092' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]': finished 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: osdmap e52: 8 total, 7 up, 8 in 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:29 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/2920597128' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/1545332092' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]': finished 2024-06-21T00:41:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: osdmap e52: 8 total, 7 up, 8 in 2024-06-21T00:41:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/2920597128' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/1545332092' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]: dispatch 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0b75dcb-fa21-40d8-bd10-977b14372daa"}]': finished 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: osdmap e52: 8 total, 7 up, 8 in 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:30.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:29 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/2920597128' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-21T00:41:31.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:30 smithi120 ceph-mon[25975]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-06-21T00:41:31.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:30 smithi027 ceph-mon[25136]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-06-21T00:41:31.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:30 smithi027 ceph-mon[29814]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-06-21T00:41:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:32 smithi120 ceph-mon[25975]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:41:33.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:32 smithi027 ceph-mon[29814]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:41:33.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:32 smithi027 ceph-mon[25136]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-21T00:41:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:34 smithi120 ceph-mon[25975]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:35.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:34 smithi027 ceph-mon[25136]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:35.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:34 smithi027 ceph-mon[29814]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:36.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:35 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T00:41:36.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:36 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:36.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T00:41:36.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:36 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T00:41:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:35 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:37.049 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:37 smithi120 ceph-mon[25975]: Deploying daemon osd.7 on smithi120 2024-06-21T00:41:37.049 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:37 smithi120 ceph-mon[25975]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-06-21T00:41:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:37 smithi027 ceph-mon[25136]: Deploying daemon osd.7 on smithi120 2024-06-21T00:41:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:37 smithi027 ceph-mon[25136]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-06-21T00:41:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:37 smithi027 ceph-mon[29814]: Deploying daemon osd.7 on smithi120 2024-06-21T00:41:37.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:37 smithi027 ceph-mon[29814]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-06-21T00:41:39.277 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:39 smithi120 ceph-mon[25975]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:41:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:39 smithi027 ceph-mon[25136]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:41:39.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:39 smithi027 ceph-mon[29814]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-21T00:41:41.324 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:41 smithi120 ceph-mon[25975]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:41.325 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:41.325 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:41.325 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:41.325 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:41 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[25136]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[29814]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:41.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:41 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:42.220 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:42 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:41:42.220 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:42 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:41:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:41:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:42 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:41:42.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:42 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:41:42.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:42 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:41:43.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:43 smithi120 ceph-mon[25975]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:43.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:43 smithi027 ceph-mon[25136]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:43 smithi027 ceph-mon[29814]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:43.723 INFO:teuthology.orchestra.run.smithi120.stdout:Created osd(s) 7 on host 'smithi120' 2024-06-21T00:41:44.339 DEBUG:teuthology.orchestra.run.smithi120:osd.7> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.7.service 2024-06-21T00:41:44.341 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-06-21T00:41:44.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd stat -f json 2024-06-21T00:41:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:44 smithi120 ceph-mon[25975]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:44 smithi120 ceph-mon[25975]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:44 smithi120 ceph-mon[25975]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:44 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:44.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[25136]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[25136]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[25136]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[29814]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[29814]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[29814]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T00:41:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:44 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:45.906 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-mon[25975]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T00:41:45.906 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-mon[25975]: osdmap e53: 8 total, 7 up, 8 in 2024-06-21T00:41:45.906 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:45.906 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-mon[25975]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:45.906 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-mon[25975]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:45.909 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[38096]: 2024-06-21T00:41:45.746+0000 7f3a279d0700 -1 osd.7 0 waiting for initial osdmap 2024-06-21T00:41:45.910 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 00:41:45 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[38096]: 2024-06-21T00:41:45.753+0000 7f3a21365700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T00:41:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[25136]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T00:41:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[25136]: osdmap e53: 8 total, 7 up, 8 in 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[25136]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[25136]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[29814]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[29814]: osdmap e53: 8 total, 7 up, 8 in 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[29814]: from='osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:45 smithi027 ceph-mon[29814]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T00:41:46.450 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: purged_snaps scrub starts 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: purged_snaps scrub ok 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: osdmap e54: 8 total, 7 up, 8 in 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1926541271' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:47.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758] boot 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: osdmap e55: 8 total, 8 up, 8 in 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: purged_snaps scrub starts 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: purged_snaps scrub ok 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: osdmap e54: 8 total, 7 up, 8 in 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1926541271' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:47.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758] boot 2024-06-21T00:41:47.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: osdmap e55: 8 total, 8 up, 8 in 2024-06-21T00:41:47.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:46 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.099 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1718930482,"num_in_osds":8,"osd_in_since":1718930489,"num_remapped_pgs":0} 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: purged_snaps scrub starts 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: purged_snaps scrub ok 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]': finished 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: osdmap e54: 8 total, 7 up, 8 in 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1926541271' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: osd.7 [v2:172.21.15.120:6824/2359599758,v1:172.21.15.120:6825/2359599758] boot 2024-06-21T00:41:47.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: osdmap e55: 8 total, 8 up, 8 in 2024-06-21T00:41:47.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:46 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:41:48.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd stat -f json 2024-06-21T00:41:48.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi120 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi120 to 4272M 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[25136]: osdmap e56: 8 total, 8 up, 8 in 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi120 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi120 to 4272M 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:48 smithi027 ceph-mon[29814]: osdmap e56: 8 total, 8 up, 8 in 2024-06-21T00:41:48.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:48.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: Detected new or changed devices on smithi120 2024-06-21T00:41:48.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi120 to 4272M 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:48.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:48.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:48 smithi120 ceph-mon[25975]: osdmap e56: 8 total, 8 up, 8 in 2024-06-21T00:41:50.035 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:49 smithi027 ceph-mon[29814]: osdmap e57: 8 total, 8 up, 8 in 2024-06-21T00:41:50.035 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:49 smithi027 ceph-mon[25136]: osdmap e57: 8 total, 8 up, 8 in 2024-06-21T00:41:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:49 smithi120 ceph-mon[25975]: osdmap e57: 8 total, 8 up, 8 in 2024-06-21T00:41:50.246 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:41:50.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[25136]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:50.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[25136]: osdmap e58: 8 total, 8 up, 8 in 2024-06-21T00:41:50.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2526891637' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:50.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[29814]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:50.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[29814]: osdmap e58: 8 total, 8 up, 8 in 2024-06-21T00:41:50.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:50 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2526891637' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:50.886 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1718930506,"num_in_osds":8,"osd_in_since":1718930489,"num_remapped_pgs":0} 2024-06-21T00:41:50.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-21T00:41:51.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:50 smithi120 ceph-mon[25975]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:51.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:50 smithi120 ceph-mon[25975]: osdmap e58: 8 total, 8 up, 8 in 2024-06-21T00:41:51.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:50 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2526891637' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-21T00:41:51.109 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:41:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:51 smithi027 ceph-mon[25136]: osdmap e59: 8 total, 8 up, 8 in 2024-06-21T00:41:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:51 smithi027 ceph-mon[29814]: osdmap e59: 8 total, 8 up, 8 in 2024-06-21T00:41:52.034 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:51 smithi120 ceph-mon[25975]: osdmap e59: 8 total, 8 up, 8 in 2024-06-21T00:41:52.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:52 smithi027 ceph-mon[25136]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:52.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:52 smithi027 ceph-mon[29814]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:52 smithi120 ceph-mon[25975]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:41:53.142 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:41:53.142 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":59,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","created":"2024-06-21T00:36:51.034274+0000","modified":"2024-06-21T00:41:50.741875+0000","last_up_change":"2024-06-21T00:41:46.731377+0000","last_in_change":"2024-06-21T00:41:29.311684+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-21T00:39:51.999551+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d721abb8-7102-4cd5-8c27-42f36634e958","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6803","nonce":2514709173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6805","nonce":2514709173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6809","nonce":2514709173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6807","nonce":2514709173}]},"public_addr":"172.21.15.27:6803/2514709173","cluster_addr":"172.21.15.27:6805/2514709173","heartbeat_back_addr":"172.21.15.27:6809/2514709173","heartbeat_front_addr":"172.21.15.27:6807/2514709173","state":["exists","up"]},{"osd":1,"uuid":"7bff4817-8c21-4a34-87ff-36001678fd2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6811","nonce":521311865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6813","nonce":521311865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6817","nonce":521311865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6815","nonce":521311865}]},"public_addr":"172.21.15.27:6811/521311865","cluster_addr":"172.21.15.27:6813/521311865","heartbeat_back_addr":"172.21.15.27:6817/521311865","heartbeat_front_addr":"172.21.15.27:6815/521311865","state":["exists","up"]},{"osd":2,"uuid":"ad97f3ad-2a49-44a2-b30b-20f05146566a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6819","nonce":2017021934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6821","nonce":2017021934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6825","nonce":2017021934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6823","nonce":2017021934}]},"public_addr":"172.21.15.27:6819/2017021934","cluster_addr":"172.21.15.27:6821/2017021934","heartbeat_back_addr":"172.21.15.27:6825/2017021934","heartbeat_front_addr":"172.21.15.27:6823/2017021934","state":["exists","up"]},{"osd":3,"uuid":"538dc8d0-a030-47be-9c0b-7006e0fb6708","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6827","nonce":3447780199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6829","nonce":3447780199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6833","nonce":3447780199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6831","nonce":3447780199}]},"public_addr":"172.21.15.27:6827/3447780199","cluster_addr":"172.21.15.27:6829/3447780199","heartbeat_back_addr":"172.21.15.27:6833/3447780199","heartbeat_front_addr":"172.21.15.27:6831/3447780199","state":["exists","up"]},{"osd":4,"uuid":"09479a17-ed8f-4454-871d-7288f3c60f8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6801","nonce":4213715894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6803","nonce":4213715894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6807","nonce":4213715894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6805","nonce":4213715894}]},"public_addr":"172.21.15.120:6801/4213715894","cluster_addr":"172.21.15.120:6803/4213715894","heartbeat_back_addr":"172.21.15.120:6807/4213715894","heartbeat_front_addr":"172.21.15.120:6805/4213715894","state":["exists","up"]},{"osd":5,"uuid":"7f10e337-c99d-4759-8ea7-ed600b3ad8e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6808","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6809","nonce":1314144105}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6810","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6811","nonce":1314144105}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6814","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6815","nonce":1314144105}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6812","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6813","nonce":1314144105}]},"public_addr":"172.21.15.120:6809/1314144105","cluster_addr":"172.21.15.120:6811/1314144105","heartbeat_back_addr":"172.21.15.120:6815/1314144105","heartbeat_front_addr":"172.21.15.120:6813/1314144105","state":["exists","up"]},{"osd":6,"uuid":"9728d5f9-851a-4109-b5de-780d4ddc63b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6816","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6817","nonce":766528121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6818","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6819","nonce":766528121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6822","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6823","nonce":766528121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6820","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6821","nonce":766528121}]},"public_addr":"172.21.15.120:6817/766528121","cluster_addr":"172.21.15.120:6819/766528121","heartbeat_back_addr":"172.21.15.120:6823/766528121","heartbeat_front_addr":"172.21.15.120:6821/766528121","state":["exists","up"]},{"osd":7,"uuid":"e0b75dcb-fa21-40d8-bd10-977b14372daa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6824","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6825","nonce":2359599758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6826","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6827","nonce":2359599758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6830","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6831","nonce":2359599758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6828","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6829","nonce":2359599758}]},"public_addr":"172.21.15.120:6825/2359599758","cluster_addr":"172.21.15.120:6827/2359599758","heartbeat_back_addr":"172.21.15.120:6831/2359599758","heartbeat_front_addr":"172.21.15.120:6829/2359599758","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:00.642862+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:24.843220+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:48.553876+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:13.810804+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:35.183503+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:57.828389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:21.122462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:44.690387+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/1623484127":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/3207268233":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6801/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:6801/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/3774793823":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/173316124":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/2256275255":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/665992674":"2024-06-22T00:37:18.850588+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-21T00:41:53.877 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-06-21T00:39:51.999551+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-06-21T00:41:53.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-06-21T00:41:54.103 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:41:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:53 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4073584666' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:41:54.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:53 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4073584666' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:41:54.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:53 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4073584666' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:41:54.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:54 smithi027 ceph-mon[25136]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-06-21T00:41:54.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:54 smithi027 ceph-mon[29814]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-06-21T00:41:55.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:54 smithi120 ceph-mon[25975]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-06-21T00:41:56.097 INFO:teuthology.orchestra.run.smithi027.stdout:pg_num: 1 2024-06-21T00:41:56.717 INFO:tasks.cephadm:Adding prometheus.a on smithi120 2024-06-21T00:41:56.717 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply prometheus '1;smithi120=a' 2024-06-21T00:41:56.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:56 smithi027 ceph-mon[25136]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-06-21T00:41:56.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:56 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3859595674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-21T00:41:56.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:56 smithi027 ceph-mon[29814]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-06-21T00:41:56.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:56 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3859595674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-21T00:41:57.030 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:56 smithi120 ceph-mon[25975]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-06-21T00:41:57.030 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:56 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3859595674' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-21T00:41:58.603 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled prometheus update... 2024-06-21T00:41:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:58.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:58 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-21T00:41:59.205 DEBUG:teuthology.orchestra.run.smithi120:prometheus.a> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service 2024-06-21T00:41:59.208 INFO:tasks.cephadm:Adding node-exporter.a on smithi027 2024-06-21T00:41:59.208 INFO:tasks.cephadm:Adding node-exporter.b on smithi120 2024-06-21T00:41:59.208 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply node-exporter '2;smithi027=a;smithi120=b' 2024-06-21T00:41:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-21T00:41:59.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' 2024-06-21T00:41:59.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:58 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-21T00:41:59.853 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-mon[25975]: from='client.34319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-mon[25975]: Saving service prometheus spec with placement smithi120=a;count:1 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-mon[25975]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-mon[25975]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ignoring --setuser ceph since I am not root 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ignoring --setgroup ceph since I am not root 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-mgr[27147]: -- 172.21.15.120:0/2504189173 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562bc4ba8340 con 0x562bc4b80800 2024-06-21T00:41:59.854 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:41:59.785+0000 7f7772b3e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:41:59.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[25136]: from='client.34319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[25136]: Saving service prometheus spec with placement smithi120=a;count:1 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[25136]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[25136]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[29814]: from='client.34319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[29814]: Saving service prometheus spec with placement smithi120=a;count:1 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[29814]: from='mgr.14150 172.21.15.27:0/1261337408' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-21T00:41:59.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mon[29814]: mgrmap e16: y(active, since 4m), standbys: x 2024-06-21T00:41:59.934 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setuser ceph since I am not root 2024-06-21T00:41:59.934 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setgroup ceph since I am not root 2024-06-21T00:41:59.934 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-mgr[25369]: -- 172.21.15.27:0/3899222197 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555959d58340 con 0x555959d30c00 2024-06-21T00:41:59.934 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:41:59.800+0000 7f6e6422a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:41:59.934 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:41:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:41:59.933+0000 7f6e6422a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:42:00.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:41:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:41:59.914+0000 7f7772b3e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:42:00.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:00.013+0000 7f7772b3e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:42:00.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:00.036+0000 7f6e6422a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:42:00.457 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:00.322+0000 7f7772b3e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:42:00.457 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:00.407+0000 7f7772b3e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:42:00.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:00.353+0000 7f6e6422a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:42:00.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:00 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:00.438+0000 7f6e6422a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:42:01.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:01.542+0000 7f7772b3e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:42:01.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:01.706+0000 7f7772b3e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:42:01.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:01.788+0000 7f7772b3e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:42:01.875 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:01.620+0000 7f6e6422a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:42:01.875 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:01.791+0000 7f6e6422a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:42:01.875 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:01.876+0000 7f6e6422a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:42:02.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:02.097+0000 7f7772b3e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:42:02.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:02.186+0000 7f7772b3e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:42:02.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:02.279+0000 7f7772b3e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:42:02.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:02.201+0000 7f6e6422a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:42:02.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:02.296+0000 7f6e6422a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:42:02.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:02.392+0000 7f6e6422a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:42:02.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:02.517+0000 7f7772b3e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:42:02.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:02.647+0000 7f6e6422a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:42:03.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:02.956+0000 7f7772b3e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:42:03.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:03.112+0000 7f6e6422a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:42:03.821 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:03.437+0000 7f7772b3e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:42:03.821 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:03.528+0000 7f7772b3e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:42:03.821 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:03.821+0000 7f7772b3e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:42:03.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:03.618+0000 7f6e6422a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:42:03.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:03.716+0000 7f6e6422a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:42:04.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:03.910+0000 7f7772b3e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:42:04.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.023+0000 7f6e6422a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:42:04.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.117+0000 7f6e6422a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:42:04.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:04.314+0000 7f7772b3e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:42:04.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:04.444+0000 7f7772b3e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:42:04.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:04.535+0000 7f7772b3e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:42:04.896 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.542+0000 7f6e6422a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:42:04.896 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.680+0000 7f6e6422a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:42:04.896 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.777+0000 7f6e6422a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:42:04.896 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.897+0000 7f6e6422a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:42:05.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:04.647+0000 7f7772b3e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:42:05.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:04.735+0000 7f7772b3e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:42:05.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:04.990+0000 7f6e6422a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:42:05.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:05.273+0000 7f7772b3e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:42:05.939 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:05.567+0000 7f6e6422a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:42:06.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:05.628+0000 7f7772b3e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: [21/Jun/2024:00:42:05] ENGINE Bus STARTING 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: CherryPy Checker: 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: The Application mounted at '' has an empty config. 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: [21/Jun/2024:00:42:05] ENGINE Serving on http://:::9283 2024-06-21T00:42:06.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: [21/Jun/2024:00:42:05] ENGINE Bus STARTED 2024-06-21T00:42:06.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:05.940+0000 7f6e6422a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: Standby manager daemon x restarted 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:42:06.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: Active manager daemon y restarted 2024-06-21T00:42:06.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: Activating manager daemon y 2024-06-21T00:42:06.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:06 smithi120 ceph-mon[25975]: osdmap e60: 8 total, 8 up, 8 in 2024-06-21T00:42:06.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: Standby manager daemon x restarted 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: Active manager daemon y restarted 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: Activating manager daemon y 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[29814]: osdmap e60: 8 total, 8 up, 8 in 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: Standby manager daemon x restarted 2024-06-21T00:42:06.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/1219553760' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: Active manager daemon y restarted 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: Activating manager daemon y 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-mon[25136]: osdmap e60: 8 total, 8 up, 8 in 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:06] ENGINE Bus STARTING 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: CherryPy Checker: 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: The Application mounted at '' has an empty config. 2024-06-21T00:42:06.872 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:42:06.873 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:06] ENGINE Serving on http://:::9283 2024-06-21T00:42:06.873 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:06] ENGINE Bus STARTED 2024-06-21T00:42:07.578 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled node-exporter update... 2024-06-21T00:42:07.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: mgrmap e17: y(active, starting, since 0.613834s), standbys: x 2024-06-21T00:42:07.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:42:07.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: mgrmap e17: y(active, starting, since 0.613834s), standbys: x 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:42:07.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: Manager daemon y is now available 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:07.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:07.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: mgrmap e17: y(active, starting, since 0.613834s), standbys: x 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:42:07.770 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: Manager daemon y is now available 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:42:07.772 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:07.772 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:42:08.165 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:07] ENGINE Bus STARTING 2024-06-21T00:42:08.165 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:07] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:42:08.165 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:42:07] ENGINE Bus STARTED 2024-06-21T00:42:08.273 DEBUG:teuthology.orchestra.run.smithi027:node-exporter.a> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service 2024-06-21T00:42:08.276 DEBUG:teuthology.orchestra.run.smithi120:node-exporter.b> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service 2024-06-21T00:42:08.278 INFO:tasks.cephadm:Adding alertmanager.a on smithi027 2024-06-21T00:42:08.278 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply alertmanager '1;smithi027=a' 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: mgrmap e18: y(active, since 1.62165s), standbys: x 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: Saving service node-exporter spec with placement smithi027=a;smithi120=b;count:2 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: mgrmap e18: y(active, since 1.62165s), standbys: x 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: Saving service node-exporter spec with placement smithi027=a;smithi120=b;count:2 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: mgrmap e18: y(active, since 1.62165s), standbys: x 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: Saving service node-exporter spec with placement smithi027=a;smithi120=b;count:2 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:08.834 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.758 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: [21/Jun/2024:00:42:07] ENGINE Bus STARTING 2024-06-21T00:42:09.759 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: [21/Jun/2024:00:42:07] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:42:09.759 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: [21/Jun/2024:00:42:07] ENGINE Bus STARTED 2024-06-21T00:42:09.759 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:09.759 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.759 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-21T00:42:09.760 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.760 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:09 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: [21/Jun/2024:00:42:07] ENGINE Bus STARTING 2024-06-21T00:42:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: [21/Jun/2024:00:42:07] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: [21/Jun/2024:00:42:07] ENGINE Bus STARTED 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: [21/Jun/2024:00:42:07] ENGINE Bus STARTING 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: [21/Jun/2024:00:42:07] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: [21/Jun/2024:00:42:07] ENGINE Bus STARTED 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:09.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: mgrmap e19: y(active, since 2s), standbys: x 2024-06-21T00:42:09.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:09.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:09 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.452 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled alertmanager update... 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: Deploying daemon prometheus.a on smithi120 2024-06-21T00:42:10.756 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: Deploying daemon prometheus.a on smithi120 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: Deploying daemon prometheus.a on smithi120 2024-06-21T00:42:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:11.184 DEBUG:teuthology.orchestra.run.smithi027:alertmanager.a> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service 2024-06-21T00:42:11.187 INFO:tasks.cephadm:Adding grafana.a on smithi120 2024-06-21T00:42:11.187 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph orch apply grafana '1;smithi120=a' 2024-06-21T00:42:11.753 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:11 smithi120 ceph-mon[25975]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:11.753 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:11 smithi120 ceph-mon[25975]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-06-21T00:42:11.753 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:11 smithi120 ceph-mon[25975]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[29814]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[29814]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-06-21T00:42:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[29814]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[25136]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[25136]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-06-21T00:42:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:11 smithi027 ceph-mon[25136]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:13.758 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:13 smithi120 ceph-mon[25975]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:13 smithi027 ceph-mon[25136]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:13 smithi027 ceph-mon[29814]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:13.986 INFO:teuthology.orchestra.run.smithi120.stdout:Scheduled grafana update... 2024-06-21T00:42:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:14 smithi120 ceph-mon[25975]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:14 smithi120 ceph-mon[25975]: Saving service grafana spec with placement smithi120=a;count:1 2024-06-21T00:42:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:14 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:14 smithi120 ceph-mon[25975]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:15.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[29814]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:15.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[29814]: Saving service grafana spec with placement smithi120=a;count:1 2024-06-21T00:42:15.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:15.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[29814]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[25136]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi120=a", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:42:15.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[25136]: Saving service grafana spec with placement smithi120=a;count:1 2024-06-21T00:42:15.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:15.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:14 smithi027 ceph-mon[25136]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:16.290 DEBUG:teuthology.orchestra.run.smithi120:grafana.a> sudo journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service 2024-06-21T00:42:16.292 INFO:tasks.cephadm:Setting up client nodes... 2024-06-21T00:42:16.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-21T00:42:17.763 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:17 smithi120 ceph-mon[25975]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:17.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:17 smithi027 ceph-mon[25136]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:17.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:17 smithi027 ceph-mon[29814]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:18.539 INFO:teuthology.orchestra.run.smithi027.stdout:[client.0] 2024-06-21T00:42:18.540 INFO:teuthology.orchestra.run.smithi027.stdout: key = AQBqzHRmxXgFIBAAayAiks8GMoL4iQlM+suE5Q== 2024-06-21T00:42:18.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:18 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:18.771 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:18 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:18.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:18.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:18.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:18.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/691972491' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:19.182 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-21T00:42:19.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-06-21T00:42:19.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-06-21T00:42:19.224 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-21T00:42:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:19 smithi027 ceph-mon[25136]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:19 smithi027 ceph-mon[29814]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:19 smithi120 ceph-mon[25975]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:21.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-mon[25975]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:21.828 INFO:teuthology.orchestra.run.smithi120.stdout:[client.1] 2024-06-21T00:42:21.828 INFO:teuthology.orchestra.run.smithi120.stdout: key = AQBtzHRmPWUcMRAAoubYqexfP3i5Hn9pBf9pqg== 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 podman[40954]: 2024-06-21 00:42:21.939486821 +0000 UTC m=+0.355688703 container start f6866a487a6f155255a2fad37458c16111d39f0a5a8e27dbe7eaf6f61de8b57d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T00:42:21.962 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T00:42:21.963 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.963Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T00:42:21.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:21 smithi027 ceph-mon[25136]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:21 smithi027 ceph-mon[29814]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:22.246 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.964Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T00:42:22.246 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.964Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T00:42:22.246 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.965Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T00:42:22.246 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.968Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.968Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.69µs 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.968Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.968Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.968Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.679µs wal_replay_duration=183.255µs total_replay_duration=226.677µs 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.969Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.969Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.969Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.983Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.969063ms db_storage=621ns remote_storage=1.065µs web_handler=324ns query_engine=506ns scrape=434.351µs scrape_sd=16.397µs notify=767ns notify_sd=1.52µs rules=13.31258ms 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:42:21.983Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:22 smithi120 bash[40954]: f6866a487a6f155255a2fad37458c16111d39f0a5a8e27dbe7eaf6f61de8b57d 2024-06-21T00:42:22.247 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:42:22 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:42:22.630 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-06-21T00:42:22.630 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-06-21T00:42:22.630 DEBUG:teuthology.orchestra.run.smithi120:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-06-21T00:42:22.666 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-06-21T00:42:22.667 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-06-21T00:42:22.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph mgr dump --format=json 2024-06-21T00:42:22.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[25136]: from='client.? 172.21.15.120:0/429952940' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[29814]: from='client.? 172.21.15.120:0/429952940' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:22.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:22 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:22.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:22 smithi120 ceph-mon[25975]: from='client.? 172.21.15.120:0/429952940' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:22 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-21T00:42:22.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:22 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-21T00:42:22.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:22 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:22.971 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:22.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:22 smithi027 systemd[1]: Starting Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:42:23.332 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:22 smithi027 bash[44509]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-06-21T00:42:23.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:23 smithi027 ceph-mon[25136]: Deploying daemon node-exporter.a on smithi027 2024-06-21T00:42:23.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:23 smithi027 ceph-mon[25136]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:23.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:23 smithi027 ceph-mon[29814]: Deploying daemon node-exporter.a on smithi027 2024-06-21T00:42:23.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:23 smithi027 ceph-mon[29814]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:23.730 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:23 smithi027 bash[44509]: Getting image source signatures 2024-06-21T00:42:23.730 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:23 smithi027 bash[44509]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-06-21T00:42:23.730 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:23 smithi027 bash[44509]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-06-21T00:42:23.730 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:23 smithi027 bash[44509]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-06-21T00:42:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:23 smithi120 ceph-mon[25975]: Deploying daemon node-exporter.a on smithi027 2024-06-21T00:42:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:23 smithi120 ceph-mon[25975]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:25.566 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:25 smithi027 bash[44509]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-06-21T00:42:25.850 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:25 smithi027 ceph-mon[25136]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:25.850 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:25 smithi027 ceph-mon[29814]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:25.851 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:25 smithi027 bash[44509]: Writing manifest to image destination 2024-06-21T00:42:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:25 smithi120 ceph-mon[25975]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:26.196 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:42:26.479 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:26 smithi027 podman[44509]: 2024-06-21 00:42:26.097950278 +0000 UTC m=+3.209041914 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-21T00:42:26.479 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:26 smithi027 podman[44509]: 2024-06-21 00:42:26.205572147 +0000 UTC m=+3.316663772 container create 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:42:26.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:26 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2473824685' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:42:26.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:26 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:26 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2473824685' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:42:26.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:26 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:27.028 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":19,"active_gid":24343,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":1067287154},{"type":"v1","addr":"172.21.15.27:6801","nonce":1067287154}]},"active_addr":"172.21.15.27:6801/1067287154","active_change":"2024-06-21T00:42:05.943992+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34325,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2823981577}]},{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":613211723}]},{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":3971766118}]},{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2573480600}]}]}} 2024-06-21T00:42:27.031 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-06-21T00:42:27.032 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-06-21T00:42:27.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-21T00:42:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:26 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2473824685' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:42:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:26 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:27.293 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 podman[44509]: 2024-06-21 00:42:27.128789756 +0000 UTC m=+4.239881384 container init 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:42:27.293 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 podman[44509]: 2024-06-21 00:42:27.132756797 +0000 UTC m=+4.243848423 container start 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:42:27.293 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.136Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.137Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=arp 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-21T00:42:27.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=edac 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-21T00:42:27.295 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=os 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-21T00:42:27.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=stat 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=time 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=uname 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-21T00:42:27.297 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-21T00:42:27.298 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[44679]: ts=2024-06-21T00:42:27.138Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-21T00:42:27.298 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 bash[44509]: 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 2024-06-21T00:42:27.298 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:42:27 smithi027 systemd[1]: Started Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:42:27.309 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:27.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:27 smithi120 ceph-mon[25975]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:27.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:27.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:42:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-mon[25136]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-mon[29814]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:28.367 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:28 smithi120 bash[41238]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-06-21T00:42:28.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:28 smithi120 ceph-mon[25975]: Deploying daemon node-exporter.b on smithi120 2024-06-21T00:42:28.867 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:28 smithi120 bash[41238]: Getting image source signatures 2024-06-21T00:42:28.867 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:28 smithi120 bash[41238]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-06-21T00:42:28.867 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:28 smithi120 bash[41238]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-06-21T00:42:28.867 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:28 smithi120 bash[41238]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-06-21T00:42:28.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:28 smithi027 ceph-mon[25136]: Deploying daemon node-exporter.b on smithi120 2024-06-21T00:42:28.888 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:28] "GET /metrics HTTP/1.1" 200 191522 "" "Prometheus/2.33.4" 2024-06-21T00:42:28.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:28 smithi027 ceph-mon[29814]: Deploying daemon node-exporter.b on smithi120 2024-06-21T00:42:29.331 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:42:29.331 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":60,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","created":"2024-06-21T00:36:51.034274+0000","modified":"2024-06-21T00:42:05.943618+0000","last_up_change":"2024-06-21T00:41:46.731377+0000","last_in_change":"2024-06-21T00:41:29.311684+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-21T00:39:51.999551+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d721abb8-7102-4cd5-8c27-42f36634e958","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6803","nonce":2514709173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6805","nonce":2514709173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6809","nonce":2514709173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6807","nonce":2514709173}]},"public_addr":"172.21.15.27:6803/2514709173","cluster_addr":"172.21.15.27:6805/2514709173","heartbeat_back_addr":"172.21.15.27:6809/2514709173","heartbeat_front_addr":"172.21.15.27:6807/2514709173","state":["exists","up"]},{"osd":1,"uuid":"7bff4817-8c21-4a34-87ff-36001678fd2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6811","nonce":521311865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6813","nonce":521311865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6817","nonce":521311865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6815","nonce":521311865}]},"public_addr":"172.21.15.27:6811/521311865","cluster_addr":"172.21.15.27:6813/521311865","heartbeat_back_addr":"172.21.15.27:6817/521311865","heartbeat_front_addr":"172.21.15.27:6815/521311865","state":["exists","up"]},{"osd":2,"uuid":"ad97f3ad-2a49-44a2-b30b-20f05146566a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6819","nonce":2017021934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6821","nonce":2017021934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6825","nonce":2017021934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6823","nonce":2017021934}]},"public_addr":"172.21.15.27:6819/2017021934","cluster_addr":"172.21.15.27:6821/2017021934","heartbeat_back_addr":"172.21.15.27:6825/2017021934","heartbeat_front_addr":"172.21.15.27:6823/2017021934","state":["exists","up"]},{"osd":3,"uuid":"538dc8d0-a030-47be-9c0b-7006e0fb6708","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6827","nonce":3447780199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6829","nonce":3447780199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6833","nonce":3447780199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6831","nonce":3447780199}]},"public_addr":"172.21.15.27:6827/3447780199","cluster_addr":"172.21.15.27:6829/3447780199","heartbeat_back_addr":"172.21.15.27:6833/3447780199","heartbeat_front_addr":"172.21.15.27:6831/3447780199","state":["exists","up"]},{"osd":4,"uuid":"09479a17-ed8f-4454-871d-7288f3c60f8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6801","nonce":4213715894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6803","nonce":4213715894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6807","nonce":4213715894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6805","nonce":4213715894}]},"public_addr":"172.21.15.120:6801/4213715894","cluster_addr":"172.21.15.120:6803/4213715894","heartbeat_back_addr":"172.21.15.120:6807/4213715894","heartbeat_front_addr":"172.21.15.120:6805/4213715894","state":["exists","up"]},{"osd":5,"uuid":"7f10e337-c99d-4759-8ea7-ed600b3ad8e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6808","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6809","nonce":1314144105}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6810","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6811","nonce":1314144105}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6814","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6815","nonce":1314144105}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6812","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6813","nonce":1314144105}]},"public_addr":"172.21.15.120:6809/1314144105","cluster_addr":"172.21.15.120:6811/1314144105","heartbeat_back_addr":"172.21.15.120:6815/1314144105","heartbeat_front_addr":"172.21.15.120:6813/1314144105","state":["exists","up"]},{"osd":6,"uuid":"9728d5f9-851a-4109-b5de-780d4ddc63b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6816","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6817","nonce":766528121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6818","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6819","nonce":766528121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6822","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6823","nonce":766528121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6820","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6821","nonce":766528121}]},"public_addr":"172.21.15.120:6817/766528121","cluster_addr":"172.21.15.120:6819/766528121","heartbeat_back_addr":"172.21.15.120:6823/766528121","heartbeat_front_addr":"172.21.15.120:6821/766528121","state":["exists","up"]},{"osd":7,"uuid":"e0b75dcb-fa21-40d8-bd10-977b14372daa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6824","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6825","nonce":2359599758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6826","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6827","nonce":2359599758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6830","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6831","nonce":2359599758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6828","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6829","nonce":2359599758}]},"public_addr":"172.21.15.120:6825/2359599758","cluster_addr":"172.21.15.120:6827/2359599758","heartbeat_back_addr":"172.21.15.120:6831/2359599758","heartbeat_front_addr":"172.21.15.120:6829/2359599758","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:00.642862+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:24.843220+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:48.553876+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:13.810804+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:35.183503+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:57.828389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:21.122462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:44.690387+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/4189965795":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/1048506719":"2024-06-22T00:42:05.943588+0000","172.21.15.27:6801/917100846":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/3645089549":"2024-06-22T00:42:05.943588+0000","172.21.15.27:6800/917100846":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/1623484127":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/1225645423":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/3207268233":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6801/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:6801/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/3774793823":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/173316124":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/2256275255":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/665992674":"2024-06-22T00:37:18.850588+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-21T00:42:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:29 smithi027 ceph-mon[25136]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:29 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3768659355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:29 smithi027 ceph-mon[29814]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3768659355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:30.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:29 smithi120 ceph-mon[25975]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:30.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:29 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3768659355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:30.174 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-06-21T00:42:30.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd dump --format=json 2024-06-21T00:42:30.367 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 bash[41238]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-06-21T00:42:30.367 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 bash[41238]: Writing manifest to image destination 2024-06-21T00:42:30.400 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:30.822 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 podman[41238]: 2024-06-21 00:42:30.422788819 +0000 UTC m=+2.349791059 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 podman[41238]: 2024-06-21 00:42:30.513164139 +0000 UTC m=+2.440166365 container create 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 podman[41238]: 2024-06-21 00:42:30.723843818 +0000 UTC m=+2.650846057 container init 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 podman[41238]: 2024-06-21 00:42:30.726738097 +0000 UTC m=+2.653740323 container start 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.731Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.731Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.731Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.731Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=arp 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-21T00:42:30.823 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=edac 2024-06-21T00:42:30.824 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-21T00:42:30.825 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=os 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=stat 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-21T00:42:30.826 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=time 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=uname 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[41287]: ts=2024-06-21T00:42:30.732Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-21T00:42:30.827 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 bash[41238]: 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d 2024-06-21T00:42:31.117 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:42:30 smithi120 systemd[1]: Started Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:42:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:31 smithi120 ceph-mon[25975]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:31 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:31 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:31 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:31 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:31.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[25136]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[29814]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:31.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:42:31.911 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:42:31.911 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:31 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:42:32.358 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:42:32.358 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":60,"fsid":"47651940-2f66-11ef-bc9d-c7b262605968","created":"2024-06-21T00:36:51.034274+0000","modified":"2024-06-21T00:42:05.943618+0000","last_up_change":"2024-06-21T00:41:46.731377+0000","last_in_change":"2024-06-21T00:41:29.311684+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-21T00:39:51.999551+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d721abb8-7102-4cd5-8c27-42f36634e958","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6803","nonce":2514709173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6805","nonce":2514709173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6809","nonce":2514709173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2514709173},{"type":"v1","addr":"172.21.15.27:6807","nonce":2514709173}]},"public_addr":"172.21.15.27:6803/2514709173","cluster_addr":"172.21.15.27:6805/2514709173","heartbeat_back_addr":"172.21.15.27:6809/2514709173","heartbeat_front_addr":"172.21.15.27:6807/2514709173","state":["exists","up"]},{"osd":1,"uuid":"7bff4817-8c21-4a34-87ff-36001678fd2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6811","nonce":521311865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6813","nonce":521311865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6817","nonce":521311865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":521311865},{"type":"v1","addr":"172.21.15.27:6815","nonce":521311865}]},"public_addr":"172.21.15.27:6811/521311865","cluster_addr":"172.21.15.27:6813/521311865","heartbeat_back_addr":"172.21.15.27:6817/521311865","heartbeat_front_addr":"172.21.15.27:6815/521311865","state":["exists","up"]},{"osd":2,"uuid":"ad97f3ad-2a49-44a2-b30b-20f05146566a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6819","nonce":2017021934}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6821","nonce":2017021934}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6825","nonce":2017021934}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":2017021934},{"type":"v1","addr":"172.21.15.27:6823","nonce":2017021934}]},"public_addr":"172.21.15.27:6819/2017021934","cluster_addr":"172.21.15.27:6821/2017021934","heartbeat_back_addr":"172.21.15.27:6825/2017021934","heartbeat_front_addr":"172.21.15.27:6823/2017021934","state":["exists","up"]},{"osd":3,"uuid":"538dc8d0-a030-47be-9c0b-7006e0fb6708","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6827","nonce":3447780199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6829","nonce":3447780199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6833","nonce":3447780199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3447780199},{"type":"v1","addr":"172.21.15.27:6831","nonce":3447780199}]},"public_addr":"172.21.15.27:6827/3447780199","cluster_addr":"172.21.15.27:6829/3447780199","heartbeat_back_addr":"172.21.15.27:6833/3447780199","heartbeat_front_addr":"172.21.15.27:6831/3447780199","state":["exists","up"]},{"osd":4,"uuid":"09479a17-ed8f-4454-871d-7288f3c60f8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6801","nonce":4213715894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6803","nonce":4213715894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6807","nonce":4213715894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":4213715894},{"type":"v1","addr":"172.21.15.120:6805","nonce":4213715894}]},"public_addr":"172.21.15.120:6801/4213715894","cluster_addr":"172.21.15.120:6803/4213715894","heartbeat_back_addr":"172.21.15.120:6807/4213715894","heartbeat_front_addr":"172.21.15.120:6805/4213715894","state":["exists","up"]},{"osd":5,"uuid":"7f10e337-c99d-4759-8ea7-ed600b3ad8e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6808","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6809","nonce":1314144105}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6810","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6811","nonce":1314144105}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6814","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6815","nonce":1314144105}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6812","nonce":1314144105},{"type":"v1","addr":"172.21.15.120:6813","nonce":1314144105}]},"public_addr":"172.21.15.120:6809/1314144105","cluster_addr":"172.21.15.120:6811/1314144105","heartbeat_back_addr":"172.21.15.120:6815/1314144105","heartbeat_front_addr":"172.21.15.120:6813/1314144105","state":["exists","up"]},{"osd":6,"uuid":"9728d5f9-851a-4109-b5de-780d4ddc63b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6816","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6817","nonce":766528121}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6818","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6819","nonce":766528121}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6822","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6823","nonce":766528121}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6820","nonce":766528121},{"type":"v1","addr":"172.21.15.120:6821","nonce":766528121}]},"public_addr":"172.21.15.120:6817/766528121","cluster_addr":"172.21.15.120:6819/766528121","heartbeat_back_addr":"172.21.15.120:6823/766528121","heartbeat_front_addr":"172.21.15.120:6821/766528121","state":["exists","up"]},{"osd":7,"uuid":"e0b75dcb-fa21-40d8-bd10-977b14372daa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6824","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6825","nonce":2359599758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6826","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6827","nonce":2359599758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6830","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6831","nonce":2359599758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6828","nonce":2359599758},{"type":"v1","addr":"172.21.15.120:6829","nonce":2359599758}]},"public_addr":"172.21.15.120:6825/2359599758","cluster_addr":"172.21.15.120:6827/2359599758","heartbeat_back_addr":"172.21.15.120:6831/2359599758","heartbeat_front_addr":"172.21.15.120:6829/2359599758","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:00.642862+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:24.843220+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:39:48.553876+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:13.810804+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:35.183503+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:40:57.828389+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:21.122462+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-21T00:41:44.690387+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/4189965795":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/1048506719":"2024-06-22T00:42:05.943588+0000","172.21.15.27:6801/917100846":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/3645089549":"2024-06-22T00:42:05.943588+0000","172.21.15.27:6800/917100846":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/1623484127":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/1225645423":"2024-06-22T00:42:05.943588+0000","172.21.15.27:0/3207268233":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6801/581000993":"2024-06-22T00:37:41.379566+0000","172.21.15.27:6800/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:6801/225303118":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/3774793823":"2024-06-22T00:37:41.379566+0000","172.21.15.27:0/173316124":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/2256275255":"2024-06-22T00:37:18.850588+0000","172.21.15.27:0/665992674":"2024-06-22T00:37:18.850588+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-21T00:42:32.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:32 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:32.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:32 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4283255504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:32.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:32 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:32.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:32 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4283255504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:32.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-06-21T00:42:32.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-06-21T00:42:32.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-06-21T00:42:32.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-06-21T00:42:32.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-06-21T00:42:32.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-06-21T00:42:32.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-06-21T00:42:32.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-06-21T00:42:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:32 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:32 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4283255504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-21T00:42:33.723 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.746 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.746 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.746 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.760 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.783 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.817 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:33.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:33 smithi027 ceph-mon[25136]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:33.877 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:33 smithi027 ceph-mon[29814]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:33 smithi120 ceph-mon[25975]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:34.255 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:35.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:35 smithi027 ceph-mon[25136]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:35.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:35 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:35.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:35 smithi027 ceph-mon[29814]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:35.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:35 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:35 smithi120 ceph-mon[25975]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:35 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:37.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:37 smithi120 ceph-mon[25975]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:37.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:42:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:37 smithi027 ceph-mon[25136]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:37 smithi027 ceph-mon[29814]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:38.955 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:38] "GET /metrics HTTP/1.1" 200 191506 "" "Prometheus/2.33.4" 2024-06-21T00:42:39.801 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:39 smithi027 ceph-mon[25136]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:39.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:39 smithi027 ceph-mon[29814]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:39.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:39 smithi120 ceph-mon[25975]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:41.662 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:41 smithi027 ceph-mon[29814]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:41.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:41 smithi027 ceph-mon[25136]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:41.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:41 smithi120 ceph-mon[25975]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:43.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[29814]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:43.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:43.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:43.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[25136]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:43.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:43.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:43 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:43 smithi120 ceph-mon[25975]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:43 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:43 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:42:44.676 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:44 smithi027 ceph-mon[25136]: Deploying daemon alertmanager.a on smithi027 2024-06-21T00:42:44.677 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:44 smithi027 ceph-mon[29814]: Deploying daemon alertmanager.a on smithi027 2024-06-21T00:42:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:44 smithi120 ceph-mon[25975]: Deploying daemon alertmanager.a on smithi027 2024-06-21T00:42:45.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:45 smithi027 ceph-mon[25136]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:45.873 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:45 smithi027 ceph-mon[29814]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:45.898 INFO:teuthology.orchestra.run.smithi027.stdout:55834574887 2024-06-21T00:42:45.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-06-21T00:42:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:45 smithi120 ceph-mon[25975]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:47.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:47 smithi027 ceph-mon[25136]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:47 smithi027 ceph-mon[29814]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:47 smithi120 ceph-mon[25975]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:42:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:48] "GET /metrics HTTP/1.1" 200 191506 "" "Prometheus/2.33.4" 2024-06-21T00:42:49.598 INFO:teuthology.orchestra.run.smithi027.stdout:133143986204 2024-06-21T00:42:49.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-06-21T00:42:49.824 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:49 smithi027 ceph-mon[25136]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:49.825 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:49 smithi027 ceph-mon[29814]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:49 smithi120 ceph-mon[25975]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:51.391 INFO:teuthology.orchestra.run.smithi027.stdout:167503724567 2024-06-21T00:42:51.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-06-21T00:42:51.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:51 smithi120 ceph-mon[25975]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:51.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:51 smithi027 ceph-mon[25136]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:51.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:51 smithi027 ceph-mon[29814]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:51.919 INFO:teuthology.orchestra.run.smithi027.stdout:34359738413 2024-06-21T00:42:51.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-06-21T00:42:52.307 INFO:teuthology.orchestra.run.smithi027.stdout:201863462930 2024-06-21T00:42:52.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-06-21T00:42:53.505 INFO:teuthology.orchestra.run.smithi027.stdout:236223201294 2024-06-21T00:42:53.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-06-21T00:42:53.878 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:53 smithi027 ceph-mon[29814]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:53.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:53 smithi027 ceph-mon[25136]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:53 smithi120 ceph-mon[25975]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:54.569 INFO:teuthology.orchestra.run.smithi027.stdout:77309411365 2024-06-21T00:42:54.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-06-21T00:42:54.616 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:54.617 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:55.021 INFO:teuthology.orchestra.run.smithi027.stdout:111669149728 2024-06-21T00:42:55.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-06-21T00:42:55.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:55 smithi027 ceph-mon[25136]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:55.879 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:55 smithi027 ceph-mon[29814]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:55 smithi120 ceph-mon[25975]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:56.970 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:56.972 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:56.972 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:57.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:57 smithi027 ceph-mon[25136]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:57.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:57 smithi027 ceph-mon[29814]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:57 smithi120 ceph-mon[25975]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:42:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:42:58.242 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:58.242 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:42:58.701 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:42:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:42:58] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-06-21T00:42:59.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:42:59 smithi027 ceph-mon[25136]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:42:59.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:42:59 smithi027 ceph-mon[29814]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:42:59 smithi120 ceph-mon[25975]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:00.958 INFO:teuthology.orchestra.run.smithi027.stdout:201863462932 2024-06-21T00:43:01.568 INFO:teuthology.orchestra.run.smithi027.stdout:167503724569 2024-06-21T00:43:01.785 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462932 for osd.6 2024-06-21T00:43:01.785 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:01 smithi120 ceph-mon[25975]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:01 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4226668051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-21T00:43:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:01 smithi027 ceph-mon[29814]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:01 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4226668051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-21T00:43:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:01 smithi027 ceph-mon[25136]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:01.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:01 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4226668051' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-21T00:43:02.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:02 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1280461498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-21T00:43:02.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:02 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1280461498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-21T00:43:02.730 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:43:02.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:02 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1280461498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-21T00:43:03.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:03 smithi027 ceph-mon[25136]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:03.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:03 smithi027 ceph-mon[29814]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:03.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:03 smithi120 ceph-mon[25975]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:04.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724569 for osd.5 2024-06-21T00:43:04.694 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:05.060 INFO:teuthology.orchestra.run.smithi027.stdout:55834574892 2024-06-21T00:43:05.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:05 smithi027 ceph-mon[25136]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:05.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:05 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3150296382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-21T00:43:05.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:05 smithi027 ceph-mon[29814]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:05.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:05 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3150296382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-21T00:43:05.845 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574887 got 55834574892 for osd.1 2024-06-21T00:43:05.845 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:06.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:05 smithi120 ceph-mon[25975]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:06.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:05 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3150296382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-21T00:43:06.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:06.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:06.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:06.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:06.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:06.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:06.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:06.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:07.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:43:07.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:07.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:43:07.862 INFO:teuthology.orchestra.run.smithi027.stdout:34359738418 2024-06-21T00:43:07.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:07 smithi027 ceph-mon[25136]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:07.863 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:07 smithi027 ceph-mon[29814]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:07 smithi120 ceph-mon[25975]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:08 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3140804043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-21T00:43:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:08 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3140804043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-21T00:43:08.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:08] "GET /metrics HTTP/1.1" 200 191507 "" "Prometheus/2.33.4" 2024-06-21T00:43:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:08 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3140804043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-21T00:43:09.126 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738418 for osd.0 2024-06-21T00:43:09.126 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:09.719 INFO:teuthology.orchestra.run.smithi027.stdout:236223201298 2024-06-21T00:43:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:09 smithi027 ceph-mon[25136]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:09 smithi027 ceph-mon[29814]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:09 smithi120 ceph-mon[25975]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:10.297 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:10.272Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-21T00:43:10.297 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:10.298Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:43:10.297 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:10.298Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:43:10.563 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:10.300Z caller=main.go:518 msg=Listening address=:9093 2024-06-21T00:43:10.563 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:10.300Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-21T00:43:10.563 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 bash[47504]: 7b7cee47a21332d0896f4cf2889cea509cc759fe0c081c78ebf9bccf495a1d6f 2024-06-21T00:43:10.563 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:10 smithi027 systemd[1]: Started Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:10.737 INFO:teuthology.orchestra.run.smithi027.stdout:111669149733 2024-06-21T00:43:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/49919814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-21T00:43:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: Deploying daemon grafana.a on smithi120 2024-06-21T00:43:10.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:10 smithi120 ceph-mon[25975]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/49919814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-21T00:43:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: Deploying daemon grafana.a on smithi120 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[25136]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/49919814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:43:10.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:10.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: Deploying daemon grafana.a on smithi120 2024-06-21T00:43:10.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:10 smithi027 ceph-mon[29814]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:11.413 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149733 for osd.3 2024-06-21T00:43:11.413 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:11.883 INFO:teuthology.orchestra.run.smithi027.stdout:77309411370 2024-06-21T00:43:11.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1188627605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-21T00:43:11.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:11.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:11 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1188627605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-21T00:43:11.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:11 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1188627605' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-21T00:43:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:12.165 INFO:teuthology.orchestra.run.smithi027.stdout:133143986209 2024-06-21T00:43:12.472 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-06-21T00:43:12.473 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:12.698 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:12.273Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000091249s 2024-06-21T00:43:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2673221334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-21T00:43:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/569703027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-21T00:43:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[29814]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:12.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2673221334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-21T00:43:12.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/569703027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-21T00:43:12.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:12 smithi027 ceph-mon[25136]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:12 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2673221334' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-21T00:43:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:12 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/569703027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-21T00:43:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:12 smithi120 ceph-mon[25975]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:13.445 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411370 for osd.2 2024-06-21T00:43:13.445 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:13.930 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986209 for osd.4 2024-06-21T00:43:13.930 DEBUG:teuthology.parallel:result is None 2024-06-21T00:43:13.931 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-06-21T00:43:13.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph pg dump --format=json 2024-06-21T00:43:14.162 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:43:15.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:15 smithi027 ceph-mon[25136]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:15.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:15 smithi027 ceph-mon[29814]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:15.885 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:43:15.886 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-06-21T00:43:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:15 smithi120 ceph-mon[25975]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:16.473 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-06-21T00:43:14.577387+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002318"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-21T00:42:06.657283+0000","last_change":"2024-06-21T00:41:50.752584+0000","last_active":"2024-06-21T00:42:06.657283+0000","last_peered":"2024-06-21T00:42:06.657283+0000","last_clean":"2024-06-21T00:42:06.657283+0000","last_became_active":"2024-06-21T00:41:50.751987+0000","last_became_peered":"2024-06-21T00:41:50.751987+0000","last_unstale":"2024-06-21T00:42:06.657283+0000","last_undegraded":"2024-06-21T00:42:06.657283+0000","last_fullsized":"2024-06-21T00:42:06.657283+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-21T00:39:52.459354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-21T00:39:52.459354+0000","last_clean_scrub_stamp":"2024-06-21T00:39:52.459354+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-22T10:46:40.831152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.42799999999999999}]},{"osd":1,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.44700000000000001}]},{"osd":2,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.39400000000000002}]},{"osd":3,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48899999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.51000000000000001}]},{"osd":1,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53800000000000003}]},{"osd":2,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.5},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56299999999999994}]},{"osd":3,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.629},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64300000000000002}]},{"osd":4,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.70899999999999996}]},{"osd":5,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.67800000000000005}]},{"osd":7,"last update":"Fri Jun 21 00:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.72899999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35599999999999998,"15min":0.33200000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.375,"15min":0.34499999999999997},"min":{"1min":0.247,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.36399999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:42:55 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44,"15min":0.42599999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.44,"15min":0.438},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.34699999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:42:21 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.33400000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.504,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39100000000000001}]},{"osd":4,"last update":"Fri Jun 21 00:42:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60299999999999998,"15min":0.60799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.748,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48699999999999999,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.88800000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.53100000000000003}]},{"osd":5,"last update":"Fri Jun 21 00:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.65200000000000002,"15min":0.65600000000000003},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.85599999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56000000000000005}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.5}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52000000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jun 21 00:42:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.26400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.36899999999999999,"15min":0.34100000000000003},"min":{"1min":0.29399999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.47899999999999998,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42799999999999999,"15min":0.41499999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.38600000000000001}]},{"osd":3,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.42599999999999999,"15min":0.42299999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.35099999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:45 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.69499999999999995}]},{"osd":5,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.623,"15min":0.63100000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.67100000000000004}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.56599999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.36599999999999999,"15min":0.36899999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.39500000000000002,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.373,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56200000000000006,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33400000000000002}]},{"osd":1,"last update":"Fri Jun 21 00:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.374,"15min":0.371},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46800000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.40600000000000003},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.35599999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:42:20 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.46300000000000002,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.38700000000000001}]},{"osd":4,"last update":"Fri Jun 21 00:42:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80100000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.60199999999999998,"15min":0.61199999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.755,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.63400000000000001}]},{"osd":5,"last update":"Fri Jun 21 00:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.76000000000000001,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70799999999999996,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61799999999999999}]},{"osd":7,"last update":"Fri Jun 21 00:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60499999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51200000000000001,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56799999999999995,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.251}]},{"osd":1,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.57399999999999995,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.58299999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.224}]},{"osd":2,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.501,"15min":0.50900000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58999999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.47999999999999998,"15min":0.48899999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.52500000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.28899999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.622,"15min":0.63200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.49299999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.73399999999999999,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.52300000000000002}]},{"osd":5,"last update":"Fri Jun 21 00:43:08 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.48799999999999999}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.54500000000000004}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.55700000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71199999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.69399999999999995}]},{"osd":1,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57699999999999996,"15min":0.57799999999999996},"min":{"1min":0.46000000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.93000000000000005}]},{"osd":2,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54000000000000004,"15min":0.53900000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69399999999999995,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.745}]},{"osd":3,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73999999999999999,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.75600000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.86599999999999999}]},{"osd":5,"last update":"Fri Jun 21 00:43:03 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.40400000000000003,"15min":0.39800000000000002},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.40799999999999997,"15min":0.39800000000000002},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.78400000000000003}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":1.097}]},{"osd":7,"last update":"Fri Jun 21 00:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":1.1279999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.81399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.51900000000000002}]},{"osd":1,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52700000000000002,"15min":0.52900000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81599999999999995,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.49099999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80600000000000005,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.47899999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57399999999999995}]},{"osd":4,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.44},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.44800000000000001,"15min":0.443},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.40600000000000003}]},{"osd":6,"last update":"Fri Jun 21 00:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-21T00:43:16.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph pg dump --format=json 2024-06-21T00:43:16.700 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:43:17.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:17 smithi027 ceph-mon[25136]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:17.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:17 smithi027 ceph-mon[25136]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:17.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:17 smithi027 ceph-mon[29814]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:17.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:17 smithi027 ceph-mon[29814]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:17 smithi120 ceph-mon[25975]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:17 smithi120 ceph-mon[25975]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:18.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:18.423 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:43:18.425 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-06-21T00:43:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:18] "GET /metrics HTTP/1.1" 200 191507 "" "Prometheus/2.33.4" 2024-06-21T00:43:19.121 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-06-21T00:43:16.577791+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002362"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-21T00:42:06.657283+0000","last_change":"2024-06-21T00:41:50.752584+0000","last_active":"2024-06-21T00:42:06.657283+0000","last_peered":"2024-06-21T00:42:06.657283+0000","last_clean":"2024-06-21T00:42:06.657283+0000","last_became_active":"2024-06-21T00:41:50.751987+0000","last_became_peered":"2024-06-21T00:41:50.751987+0000","last_unstale":"2024-06-21T00:42:06.657283+0000","last_undegraded":"2024-06-21T00:42:06.657283+0000","last_fullsized":"2024-06-21T00:42:06.657283+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-21T00:39:52.459354+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-21T00:39:52.459354+0000","last_clean_scrub_stamp":"2024-06-21T00:39:52.459354+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-22T10:46:40.831152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.42799999999999999}]},{"osd":1,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.44700000000000001}]},{"osd":2,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.39400000000000002}]},{"osd":3,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Fri Jun 21 00:42:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48899999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.51000000000000001}]},{"osd":1,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53800000000000003}]},{"osd":2,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.5},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56299999999999994}]},{"osd":3,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.629},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64300000000000002}]},{"osd":4,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.70899999999999996}]},{"osd":5,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.67800000000000005}]},{"osd":7,"last update":"Fri Jun 21 00:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.72899999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:30 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35599999999999998,"15min":0.33200000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.375,"15min":0.34499999999999997},"min":{"1min":0.247,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.36399999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:42:55 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44,"15min":0.42599999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.44,"15min":0.438},"min":{"1min":0.30399999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.34699999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:42:21 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.33400000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.504,"15min":0.498},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.39100000000000001}]},{"osd":4,"last update":"Fri Jun 21 00:42:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60299999999999998,"15min":0.60799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.748,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48699999999999999,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.88800000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.53100000000000003}]},{"osd":5,"last update":"Fri Jun 21 00:43:04 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.65200000000000002,"15min":0.65600000000000003},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.85599999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56000000000000005}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.5}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52000000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jun 21 00:42:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.26400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.36899999999999999,"15min":0.34100000000000003},"min":{"1min":0.29399999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.47899999999999998,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.42799999999999999,"15min":0.41499999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.38600000000000001}]},{"osd":3,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.42599999999999999,"15min":0.42299999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.35099999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:45 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.69499999999999995}]},{"osd":5,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.623,"15min":0.63100000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.67100000000000004}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.56599999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.36599999999999999,"15min":0.36899999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.39500000000000002,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.373,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56200000000000006,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33400000000000002}]},{"osd":1,"last update":"Fri Jun 21 00:42:54 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.374,"15min":0.371},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46800000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.40600000000000003},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.71399999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.35599999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:42:20 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.46300000000000002,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.38700000000000001}]},{"osd":4,"last update":"Fri Jun 21 00:42:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.80100000000000005,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.60199999999999998,"15min":0.61199999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.755,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.63400000000000001}]},{"osd":5,"last update":"Fri Jun 21 00:43:01 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.76000000000000001,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70799999999999996,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Jun 21 00:42:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61799999999999999}]},{"osd":7,"last update":"Fri Jun 21 00:42:49 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60499999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51200000000000001,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.56799999999999995,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.251}]},{"osd":1,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.57399999999999995,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.58299999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.224}]},{"osd":2,"last update":"Fri Jun 21 00:42:17 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.501,"15min":0.50900000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.58999999999999997,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.47999999999999998,"15min":0.48899999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.52500000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.28899999999999998}]},{"osd":4,"last update":"Fri Jun 21 00:42:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.622,"15min":0.63200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.49299999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.73399999999999999,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.52300000000000002}]},{"osd":5,"last update":"Fri Jun 21 00:43:08 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.48799999999999999}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.54500000000000004}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.55700000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71199999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.69399999999999995}]},{"osd":1,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57699999999999996,"15min":0.57799999999999996},"min":{"1min":0.46000000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70899999999999996,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.93000000000000005}]},{"osd":2,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54000000000000004,"15min":0.53900000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69399999999999995,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.745}]},{"osd":3,"last update":"Fri Jun 21 00:42:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73999999999999999,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.75600000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.86599999999999999}]},{"osd":5,"last update":"Fri Jun 21 00:43:03 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.40400000000000003,"15min":0.39800000000000002},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.40799999999999997,"15min":0.39800000000000002},"min":{"1min":0.30199999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.78400000000000003}]},{"osd":6,"last update":"Fri Jun 21 00:42:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":1.097}]},{"osd":7,"last update":"Fri Jun 21 00:42:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":1.1279999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.81399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.35499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.51900000000000002}]},{"osd":1,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.52700000000000002,"15min":0.52900000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81599999999999995,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.49099999999999999}]},{"osd":2,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.80600000000000005,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.47899999999999998}]},{"osd":3,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57399999999999995}]},{"osd":4,"last update":"Fri Jun 21 00:43:02 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.44},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.44800000000000001,"15min":0.443},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.40600000000000003}]},{"osd":6,"last update":"Fri Jun 21 00:42:27 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42899999999999999}]},{"osd":7,"last update":"Fri Jun 21 00:42:48 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-21T00:43:19.123 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-06-21T00:43:19.123 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-06-21T00:43:19.123 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-06-21T00:43:19.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- ceph health --format=json 2024-06-21T00:43:19.361 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/mon.a/config 2024-06-21T00:43:19.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:19 smithi027 ceph-mon[25136]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:19.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:19 smithi027 ceph-mon[25136]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:19.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:19 smithi027 ceph-mon[29814]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:19.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:19 smithi027 ceph-mon[29814]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:19 smithi120 ceph-mon[25975]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-21T00:43:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:19 smithi120 ceph-mon[25975]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:20.572 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:20 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:20.275Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002274432s 2024-06-21T00:43:21.149 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:43:21.149 INFO:teuthology.orchestra.run.smithi027.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-06-21T00:43:21.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:21 smithi027 ceph-mon[25136]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:21.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:21 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2817580423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-21T00:43:21.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:21 smithi027 ceph-mon[29814]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:21.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:21 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2817580423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-21T00:43:21.791 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-06-21T00:43:21.791 INFO:tasks.cephadm:Setup complete, yielding 2024-06-21T00:43:21.791 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-21T00:43:21.801 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-06-21T00:43:21.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-06-21T00:43:21.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:21 smithi120 ceph-mon[25975]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:21.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:21 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2817580423' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-21T00:43:23.118 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 podman[41961]: 2024-06-21 00:43:22.96384061 +0000 UTC m=+0.017189296 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-06-21T00:43:23.406 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 podman[41961]: 2024-06-21 00:43:23.28056992 +0000 UTC m=+0.333918603 container init da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , distribution-scope=public, version=8.5, vendor=Red Hat, Inc., name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 podman[41961]: 2024-06-21 00:43:23.284185011 +0000 UTC m=+0.337533695 container start da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, description=Ceph Grafana Container, release=236.1648460182, io.openshift.expose-services=, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, version=8.5, vendor=Red Hat, Inc.) 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-06-21T00:43:23.407 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="App mode production" logger=settings 2024-06-21T00:43:23.408 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-06-21T00:43:23.409 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-06-21T00:43:23.410 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-06-21T00:43:23.411 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-06-21T00:43:23.412 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-06-21T00:43:23.413 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-06-21T00:43:23.413 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-06-21T00:43:23.413 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-06-21T00:43:23.413 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-06-21T00:43:23.413 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-06-21T00:43:23.414 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-06-21T00:43:23.415 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 bash[41961]: da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 2024-06-21T00:43:23.416 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 systemd[1]: Started Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-06-21T00:43:23.417 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-06-21T00:43:23.418 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-06-21T00:43:23.657 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-06-21T00:43:23.657 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-06-21T00:43:23.657 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-06-21T00:43:23.658 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-06-21T00:43:23.659 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-06-21T00:43:23.660 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-06-21T00:43:23.661 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-06-21T00:43:23.662 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-06-21T00:43:23.662 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-06-21T00:43:23.662 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-06-21T00:43:23.662 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-06-21T00:43:23.663 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-06-21T00:43:23.664 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-06-21T00:43:23.665 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-06-21T00:43:23.666 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-06-21T00:43:23.667 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-06-21T00:43:23.668 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-06-21T00:43:23.669 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-06-21T00:43:23.670 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-06-21T00:43:23.671 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-06-21T00:43:23.672 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-06-21T00:43:23.673 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-06-21T00:43:23.674 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-06-21T00:43:23.675 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-06-21T00:43:24.012 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-06-21T00:43:24.013 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-06-21T00:43:24.014 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-06-21T00:43:24.014 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-06-21T00:43:24.014 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-06-21T00:43:24.014 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-06-21T00:43:24.014 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-06-21T00:43:24.015 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-06-21T00:43:24.016 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-06-21T00:43:24.017 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-06-21T00:43:24.018 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=439.071692ms 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-06-21T00:43:24.019 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:43:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:43:23+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-06-21T00:43:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[25136]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[29814]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:24 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:24 smithi120 ceph-mon[25975]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:24 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:24 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:24 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:24 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "5edb397f-9351-4be1-9934-ca84709cc93d", 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "r", 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout: "current_period": "9aa5fc19-9644-4f18-8c77-5e56ac5058b6", 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout: "epoch": 1 2024-06-21T00:43:25.484 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:43:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[29814]: osdmap e61: 8 total, 8 up, 8 in 2024-06-21T00:43:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4285980052' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[29814]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:25.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[25136]: osdmap e61: 8 total, 8 up, 8 in 2024-06-21T00:43:25.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4285980052' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:25 smithi027 ceph-mon[25136]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:25.784 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:25 smithi120 ceph-mon[25975]: osdmap e61: 8 total, 8 up, 8 in 2024-06-21T00:43:25.785 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:25 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4285980052' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.785 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:25 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-06-21T00:43:25.785 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:25 smithi120 ceph-mon[25975]: pgmap v43: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:26.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:26 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-21T00:43:26.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:26 smithi120 ceph-mon[25975]: osdmap e62: 8 total, 8 up, 8 in 2024-06-21T00:43:26.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:26 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:26.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-21T00:43:26.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[25136]: osdmap e62: 8 total, 8 up, 8 in 2024-06-21T00:43:26.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-06-21T00:43:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[29814]: osdmap e62: 8 total, 8 up, 8 in 2024-06-21T00:43:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:26 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:26.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-06-21T00:43:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[29814]: osdmap e63: 8 total, 8 up, 8 in 2024-06-21T00:43:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[29814]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[25136]: osdmap e63: 8 total, 8 up, 8 in 2024-06-21T00:43:27.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[25136]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:27.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.816 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-mon[25975]: osdmap e63: 8 total, 8 up, 8 in 2024-06-21T00:43:27.816 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-mon[25975]: pgmap v46: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:27.816 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.816 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:27.816 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:28.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:28 smithi027 ceph-mon[25136]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:43:28.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:28 smithi027 ceph-mon[25136]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:43:28.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:28] "GET /metrics HTTP/1.1" 200 191505 "" "Prometheus/2.33.4" 2024-06-21T00:43:28.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:28 smithi027 ceph-mon[29814]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:43:28.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:28 smithi027 ceph-mon[29814]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:43:28.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:28 smithi120 ceph-mon[25975]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:43:28.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:28 smithi120 ceph-mon[25975]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "6e821188-4b87-4bf6-9eda-107a69e594a4", 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default", 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "api_name": "default", 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "is_master": "true", 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-06-21T00:43:29.563 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames": [], 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames_s3website": [], 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "", 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "zones": [], 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_targets": [], 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "default_placement": "", 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "5edb397f-9351-4be1-9934-ca84709cc93d", 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_policy": { 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: "groups": [] 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:29.564 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:43:29.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:29 smithi027 ceph-mon[25136]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:29 smithi027 ceph-mon[29814]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:29.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:29 smithi120 ceph-mon[25975]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:30.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-06-21T00:43:31.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:31 smithi027 ceph-mon[25136]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:31.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:31 smithi027 ceph-mon[29814]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:31 smithi120 ceph-mon[25975]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-06-21T00:43:32.479 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:32 smithi027 systemd[1]: Stopping Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:43:33.612 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:33 smithi027 bash[49231]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-21T00:43:33.612 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[47695]: level=info ts=2024-06-21T00:43:33.483Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-21T00:43:33.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:33 smithi027 ceph-mon[25136]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-06-21T00:43:33.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:33 smithi027 ceph-mon[29814]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-06-21T00:43:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:33 smithi120 ceph-mon[25975]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "e42ab059-5e03-4511-8024-81ef34ca208f", 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "z", 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "domain_root": "z.rgw.meta:root", 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "control_pool": "z.rgw.control", 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "gc_pool": "z.rgw.log:gc", 2024-06-21T00:43:34.123 INFO:teuthology.orchestra.run.smithi027.stdout: "lc_pool": "z.rgw.log:lc", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "log_pool": "z.rgw.log", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "roles_pool": "z.rgw.meta:roles", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "otp_pool": "z.rgw.otp", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "system_key": { 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "access_key": "", 2024-06-21T00:43:34.124 INFO:teuthology.orchestra.run.smithi027.stdout: "secret_key": "" 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_pools": [ 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "key": "default-placement", 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "val": { 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "index_pool": "z.rgw.buckets.index", 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "storage_classes": { 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "STANDARD": { 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "data_pool": "z.rgw.buckets.data" 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-06-21T00:43:34.125 INFO:teuthology.orchestra.run.smithi027.stdout: "index_type": 0 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "5edb397f-9351-4be1-9934-ca84709cc93d", 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout: "notif_pool": "z.rgw.log:notif" 2024-06-21T00:43:34.126 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:43:34.595 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:34 smithi027 podman[49264]: 2024-06-21 00:43:34.333418102 +0000 UTC m=+0.960725398 container died 7b7cee47a21332d0896f4cf2889cea509cc759fe0c081c78ebf9bccf495a1d6f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:34.864 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:34 smithi027 podman[49264]: 2024-06-21 00:43:34.695537628 +0000 UTC m=+1.322844924 container cleanup 7b7cee47a21332d0896f4cf2889cea509cc759fe0c081c78ebf9bccf495a1d6f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:34.864 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:34 smithi027 bash[49264]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-21T00:43:34.864 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:34 smithi027 bash[49390]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-21T00:43:35.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-06-21T00:43:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:35 smithi027 ceph-mon[25136]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-06-21T00:43:35.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:35 smithi027 ceph-mon[29814]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-06-21T00:43:35.730 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:35 smithi027 podman[49297]: 2024-06-21 00:43:35.465399724 +0000 UTC m=+1.958704473 container remove 7b7cee47a21332d0896f4cf2889cea509cc759fe0c081c78ebf9bccf495a1d6f (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:35.730 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:35 smithi027 podman[49297]: 2024-06-21 00:43:35.558358066 +0000 UTC m=+2.051662827 volume remove b19a61f0191b33ed13381b3e4d5c89762422c09f633a738a35ca8a8297f2737f 2024-06-21T00:43:35.730 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:35 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-21T00:43:35.730 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:35 smithi027 systemd[1]: Stopped Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:35.730 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:35 smithi027 systemd[1]: Starting Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:43:36.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:35 smithi120 ceph-mon[25975]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 448 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-06-21T00:43:36.891 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:36 smithi027 podman[49534]: 2024-06-21 00:43:36.665679747 +0000 UTC m=+0.023779827 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-06-21T00:43:37.229 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 podman[49534]: 2024-06-21 00:43:37.017824906 +0000 UTC m=+0.375925001 volume create ad25015aec00368d0ebac31712b53d7a5a38654bf720c79a54dd580842c6cda9 2024-06-21T00:43:37.229 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 podman[49534]: 2024-06-21 00:43:37.093056039 +0000 UTC m=+0.451156088 container create aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:37.640 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 podman[49534]: 2024-06-21 00:43:37.395828633 +0000 UTC m=+0.753928692 container init aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:37.640 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 podman[49534]: 2024-06-21 00:43:37.398640517 +0000 UTC m=+0.756740567 container start aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:37.640 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.409Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.409Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.410Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.411Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.437Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.437Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.439Z caller=main.go:518 msg=Listening address=:9093 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:37.439Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 bash[49534]: aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 2024-06-21T00:43:37.641 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:37 smithi027 systemd[1]: Started Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:37.815 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:37 smithi120 ceph-mon[25975]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 404 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-06-21T00:43:37.815 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:37 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-mon[25136]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 404 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-06-21T00:43:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-mon[29814]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 404 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-06-21T00:43:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:37 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:38 smithi027 ceph-mon[25136]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:43:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:38 smithi027 ceph-mon[25136]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:43:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:38 smithi027 ceph-mon[29814]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:43:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:38 smithi027 ceph-mon[29814]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:43:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:38] "GET /metrics HTTP/1.1" 200 193110 "" "Prometheus/2.33.4" 2024-06-21T00:43:39.015 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:38 smithi120 ceph-mon[25975]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:43:39.015 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:38 smithi120 ceph-mon[25975]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:43:39.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:39.412Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000446043s 2024-06-21T00:43:39.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[25136]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-21T00:43:39.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[25136]: osdmap e64: 8 total, 8 up, 8 in 2024-06-21T00:43:39.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-21T00:43:39.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[29814]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-21T00:43:39.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[29814]: osdmap e64: 8 total, 8 up, 8 in 2024-06-21T00:43:39.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:39 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-21T00:43:40.023 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:39 smithi120 ceph-mon[25975]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-21T00:43:40.023 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:39 smithi120 ceph-mon[25975]: osdmap e64: 8 total, 8 up, 8 in 2024-06-21T00:43:40.024 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:39 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 bash[42641]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T00:43:40.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[40970]: ts=2024-06-21T00:43:40.536Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T00:43:40.619 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 podman[42654]: 2024-06-21 00:43:40.546358377 +0000 UTC m=+0.129946381 container died f6866a487a6f155255a2fad37458c16111d39f0a5a8e27dbe7eaf6f61de8b57d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:40.925 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-21T00:43:40.925 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:40 smithi120 ceph-mon[25975]: osdmap e65: 8 total, 8 up, 8 in 2024-06-21T00:43:40.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:40 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-21T00:43:40.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:40 smithi027 ceph-mon[25136]: osdmap e65: 8 total, 8 up, 8 in 2024-06-21T00:43:40.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:40 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-06-21T00:43:40.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:40 smithi027 ceph-mon[29814]: osdmap e65: 8 total, 8 up, 8 in 2024-06-21T00:43:41.193 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 podman[42654]: 2024-06-21 00:43:40.927289204 +0000 UTC m=+0.510877204 container cleanup f6866a487a6f155255a2fad37458c16111d39f0a5a8e27dbe7eaf6f61de8b57d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:41.193 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 bash[42654]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T00:43:41.193 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:40 smithi120 bash[42681]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 podman[42667]: 2024-06-21 00:43:41.195139172 +0000 UTC m=+0.646941968 container remove f6866a487a6f155255a2fad37458c16111d39f0a5a8e27dbe7eaf6f61de8b57d (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 podman[42742]: 2024-06-21 00:43:41.332342467 +0000 UTC m=+0.018019457 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-21T00:43:41.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 podman[42742]: 2024-06-21 00:43:41.437690402 +0000 UTC m=+0.123367435 container create d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:41.918 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-mon[25975]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 639 B/s wr, 1 op/s 2024-06-21T00:43:41.918 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-mon[25975]: osdmap e66: 8 total, 8 up, 8 in 2024-06-21T00:43:41.918 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-21T00:43:41.918 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 podman[42742]: 2024-06-21 00:43:41.69035905 +0000 UTC m=+0.376036040 container init d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 podman[42742]: 2024-06-21 00:43:41.693208677 +0000 UTC m=+0.378885676 container start d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.716Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.718Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.718Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.721Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.726Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.726Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.586µs 2024-06-21T00:43:41.919 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.726Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T00:43:41.920 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 bash[42742]: d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 2024-06-21T00:43:41.921 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:43:41.921 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.919Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[25136]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 639 B/s wr, 1 op/s 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[25136]: osdmap e66: 8 total, 8 up, 8 in 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[29814]: pgmap v55: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 639 B/s wr, 1 op/s 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[29814]: osdmap e66: 8 total, 8 up, 8 in 2024-06-21T00:43:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:41 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-06-21T00:43:42.283 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.920Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.920Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.383µs wal_replay_duration=194.318308ms total_replay_duration=194.384178ms 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.923Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.923Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.923Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.937Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.933526ms db_storage=748ns remote_storage=1.086µs web_handler=641ns query_engine=690ns scrape=376.661µs scrape_sd=22.875µs notify=19.851µs notify_sd=5.868µs rules=13.293511ms 2024-06-21T00:43:42.284 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:43:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:43:41.937Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T00:43:42.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-21T00:43:42.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: osdmap e67: 8 total, 8 up, 8 in 2024-06-21T00:43:42.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: osdmap e67: 8 total, 8 up, 8 in 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:42.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:42.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:42 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:43.000 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3816776211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: osdmap e67: 8 total, 8 up, 8 in 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:43.001 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:43.002 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:42 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:43.954 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:43.954 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:43.954 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:43.954 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: pgmap v58: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: osdmap e68: 8 total, 8 up, 8 in 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-21T00:43:43.955 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:43 smithi120 ceph-mon[25975]: osdmap e69: 8 total, 8 up, 8 in 2024-06-21T00:43:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: pgmap v58: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: osdmap e68: 8 total, 8 up, 8 in 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[25136]: osdmap e69: 8 total, 8 up, 8 in 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.27:9093"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.120:9095"}]: dispatch 2024-06-21T00:43:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.120:3000"}]: dispatch 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: pgmap v58: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: osdmap e68: 8 total, 8 up, 8 in 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-06-21T00:43:43.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:43 smithi027 ceph-mon[29814]: osdmap e69: 8 total, 8 up, 8 in 2024-06-21T00:43:44.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-21T00:43:44.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-21T00:43:44.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[29814]: osdmap e70: 8 total, 8 up, 8 in 2024-06-21T00:43:44.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-21T00:43:44.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-21T00:43:44.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-21T00:43:44.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[25136]: osdmap e70: 8 total, 8 up, 8 in 2024-06-21T00:43:44.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:44 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-21T00:43:45.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:44 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-06-21T00:43:45.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:44 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-06-21T00:43:45.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:44 smithi120 ceph-mon[25975]: osdmap e70: 8 total, 8 up, 8 in 2024-06-21T00:43:45.101 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:44 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-06-21T00:43:45.732 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "1e988603-6f3a-46d1-a64d-2981ec3f3fb4", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "epoch": 1, 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "predecessor_uuid": "9aa5fc19-9644-4f18-8c77-5e56ac5058b6", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_status": [], 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "period_map": { 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "1e988603-6f3a-46d1-a64d-2981ec3f3fb4", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "zonegroups": [ 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "6e821188-4b87-4bf6-9eda-107a69e594a4", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "api_name": "default", 2024-06-21T00:43:45.733 INFO:teuthology.orchestra.run.smithi027.stdout: "is_master": "true", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames": [], 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames_s3website": [], 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "e42ab059-5e03-4511-8024-81ef34ca208f", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "zones": [ 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "e42ab059-5e03-4511-8024-81ef34ca208f", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "z", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "log_meta": "false", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "log_data": "false", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "bucket_index_max_shards": 11, 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "read_only": "false", 2024-06-21T00:43:45.734 INFO:teuthology.orchestra.run.smithi027.stdout: "tier_type": "", 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_from_all": "true", 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_from": [], 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "redirect_zone": "" 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_targets": [ 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default-placement", 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "tags": [], 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "storage_classes": [ 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: "STANDARD" 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.735 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "default_placement": "default-placement", 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "5edb397f-9351-4be1-9934-ca84709cc93d", 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_policy": { 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "groups": [] 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "short_zone_ids": [ 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "key": "e42ab059-5e03-4511-8024-81ef34ca208f", 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "val": 3002732996 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zonegroup": "6e821188-4b87-4bf6-9eda-107a69e594a4", 2024-06-21T00:43:45.736 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "e42ab059-5e03-4511-8024-81ef34ca208f", 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "period_config": { 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "bucket_quota": { 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "check_on_raw": false, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size": -1, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size_kb": 0, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_objects": -1 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "user_quota": { 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "check_on_raw": false, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size": -1, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size_kb": 0, 2024-06-21T00:43:45.737 INFO:teuthology.orchestra.run.smithi027.stdout: "max_objects": -1 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "user_ratelimit": { 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_ops": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_ops": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_bytes": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_bytes": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "bucket_ratelimit": { 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_ops": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_ops": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_bytes": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_bytes": 0, 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false 2024-06-21T00:43:45.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "anonymous_ratelimit": { 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_ops": 0, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_ops": 0, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "max_read_bytes": 0, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "max_write_bytes": 0, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "5edb397f-9351-4be1-9934-ca84709cc93d", 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_name": "r", 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_epoch": 2 2024-06-21T00:43:45.739 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-21T00:43:45.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:45 smithi120 ceph-mon[25975]: osdmap e71: 8 total, 8 up, 8 in 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[25136]: osdmap e71: 8 total, 8 up, 8 in 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: pgmap v61: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 6.0 KiB/s rd, 511 B/s wr, 7 op/s 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1055580716' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-06-21T00:43:45.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:45 smithi027 ceph-mon[29814]: osdmap e71: 8 total, 8 up, 8 in 2024-06-21T00:43:46.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-06-21T00:43:47.479 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:43:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:43:47.414Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002681014s 2024-06-21T00:43:47.857 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:47 smithi027 ceph-mon[25136]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:47.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:47 smithi027 ceph-mon[29814]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:47 smithi120 ceph-mon[25975]: pgmap v64: 129 pgs: 32 unknown, 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:48.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:48.646 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled rgw.foo update... 2024-06-21T00:43:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:48] "GET /metrics HTTP/1.1" 200 193110 "" "Prometheus/2.33.4" 2024-06-21T00:43:49.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph osd pool create foo' 2024-06-21T00:43:49.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[25136]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-06-21T00:43:49.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[25136]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:43:49.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[25136]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:49.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:49.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[29814]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-06-21T00:43:49.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[29814]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:43:49.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[29814]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:49.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:49 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:49 smithi120 ceph-mon[25975]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-06-21T00:43:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:49 smithi120 ceph-mon[25975]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:43:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:49 smithi120 ceph-mon[25975]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:49 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:51.863 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'foo' created 2024-06-21T00:43:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:51 smithi120 ceph-mon[25975]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-06-21T00:43:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:51 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-21T00:43:52.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:51 smithi027 ceph-mon[25136]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-06-21T00:43:52.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:51 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-21T00:43:52.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:51 smithi027 ceph-mon[29814]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 2 op/s 2024-06-21T00:43:52.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:51 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-06-21T00:43:52.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'rbd pool init foo' 2024-06-21T00:43:53.092 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-21T00:43:53.092 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[25136]: osdmap e72: 8 total, 8 up, 8 in 2024-06-21T00:43:53.093 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[25136]: pgmap v68: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 905 B/s wr, 2 op/s 2024-06-21T00:43:53.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-21T00:43:53.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[29814]: osdmap e72: 8 total, 8 up, 8 in 2024-06-21T00:43:53.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:52 smithi027 ceph-mon[29814]: pgmap v68: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 905 B/s wr, 2 op/s 2024-06-21T00:43:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:52 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1772351795' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-06-21T00:43:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:52 smithi120 ceph-mon[25975]: osdmap e72: 8 total, 8 up, 8 in 2024-06-21T00:43:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:52 smithi120 ceph-mon[25975]: pgmap v68: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 905 B/s rd, 905 B/s wr, 2 op/s 2024-06-21T00:43:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: osdmap e73: 8 total, 8 up, 8 in 2024-06-21T00:43:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: osdmap e73: 8 total, 8 up, 8 in 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:53.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:53 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: osdmap e73: 8 total, 8 up, 8 in 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:54.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:54.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:54.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:54.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:53 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:54.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[25136]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[25136]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[25136]: osdmap e74: 8 total, 8 up, 8 in 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[25136]: pgmap v71: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[29814]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[29814]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T00:43:54.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[29814]: osdmap e74: 8 total, 8 up, 8 in 2024-06-21T00:43:54.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-21T00:43:54.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:54 smithi027 ceph-mon[29814]: pgmap v71: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:55.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:54 smithi120 ceph-mon[25975]: Saving service rgw.foo spec with placement count:2 2024-06-21T00:43:55.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:54 smithi120 ceph-mon[25975]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T00:43:55.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:54 smithi120 ceph-mon[25975]: osdmap e74: 8 total, 8 up, 8 in 2024-06-21T00:43:55.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:54 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-06-21T00:43:55.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:54 smithi120 ceph-mon[25975]: pgmap v71: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:56.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:55 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-21T00:43:56.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:55 smithi027 ceph-mon[29814]: osdmap e75: 8 total, 8 up, 8 in 2024-06-21T00:43:56.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:55 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-21T00:43:56.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:55 smithi027 ceph-mon[25136]: osdmap e75: 8 total, 8 up, 8 in 2024-06-21T00:43:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:55 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2350305839' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-06-21T00:43:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:55 smithi120 ceph-mon[25975]: osdmap e75: 8 total, 8 up, 8 in 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: osdmap e76: 8 total, 8 up, 8 in 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: pgmap v74: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:57.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:56 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:57.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: osdmap e76: 8 total, 8 up, 8 in 2024-06-21T00:43:57.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: pgmap v74: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:57.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: osdmap e76: 8 total, 8 up, 8 in 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: pgmap v74: 161 pgs: 32 creating+peering, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T00:43:57.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-21T00:43:57.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:43:57.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:56 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:43:57.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch apply iscsi foo u p' 2024-06-21T00:43:58.109 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:57 smithi027 ceph-mon[25136]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T00:43:58.109 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:57 smithi027 ceph-mon[25136]: osdmap e77: 8 total, 8 up, 8 in 2024-06-21T00:43:58.109 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:57 smithi027 ceph-mon[29814]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T00:43:58.109 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:57 smithi027 ceph-mon[29814]: osdmap e77: 8 total, 8 up, 8 in 2024-06-21T00:43:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:57 smithi120 ceph-mon[25975]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T00:43:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:57 smithi120 ceph-mon[25975]: osdmap e77: 8 total, 8 up, 8 in 2024-06-21T00:43:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:43:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:43:58.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:43:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:43:58] "GET /metrics HTTP/1.1" 200 196872 "" "Prometheus/2.33.4" 2024-06-21T00:43:59.043 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:43:58 smithi027 ceph-mon[25136]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 7.4 KiB/s wr, 315 op/s 2024-06-21T00:43:59.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:43:58 smithi027 ceph-mon[29814]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 7.4 KiB/s wr, 315 op/s 2024-06-21T00:43:59.115 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:43:58 smithi120 ceph-mon[25975]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 7.4 KiB/s wr, 315 op/s 2024-06-21T00:43:59.535 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled iscsi.foo update... 2024-06-21T00:44:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: Saving service iscsi.foo spec with placement count:1 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: Saving service iscsi.foo spec with placement count:1 2024-06-21T00:44:00.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:00 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.610 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: Saving service iscsi.foo spec with placement count:1 2024-06-21T00:44:00.611 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:00 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:00.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T00:44:01.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:01 smithi120 ceph-mon[25975]: osdmap e78: 8 total, 8 up, 8 in 2024-06-21T00:44:01.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:01 smithi120 ceph-mon[25975]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 6.1 KiB/s wr, 260 op/s 2024-06-21T00:44:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:01 smithi027 ceph-mon[25136]: osdmap e78: 8 total, 8 up, 8 in 2024-06-21T00:44:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:01 smithi027 ceph-mon[25136]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 6.1 KiB/s wr, 260 op/s 2024-06-21T00:44:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:01 smithi027 ceph-mon[29814]: osdmap e78: 8 total, 8 up, 8 in 2024-06-21T00:44:01.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:01 smithi027 ceph-mon[29814]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 6.1 KiB/s wr, 260 op/s 2024-06-21T00:44:02.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:02 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:02.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:02 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:02.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:02 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:02.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:02 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:02.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:02 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:02.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:02 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:03.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:03 smithi027 ceph-mon[25136]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 259 KiB/s rd, 5.4 KiB/s wr, 464 op/s 2024-06-21T00:44:03.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:03 smithi027 ceph-mon[29814]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 259 KiB/s rd, 5.4 KiB/s wr, 464 op/s 2024-06-21T00:44:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:03 smithi120 ceph-mon[25975]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 259 KiB/s rd, 5.4 KiB/s wr, 464 op/s 2024-06-21T00:44:05.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: Checking pool "foo" exists for service iscsi.foo 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[29814]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 217 KiB/s rd, 4.5 KiB/s wr, 389 op/s 2024-06-21T00:44:05.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: Checking pool "foo" exists for service iscsi.foo 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:44:05.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:05 smithi027 ceph-mon[25136]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 217 KiB/s rd, 4.5 KiB/s wr, 389 op/s 2024-06-21T00:44:05.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:05.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: Checking pool "foo" exists for service iscsi.foo 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:44:05.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:05 smithi120 ceph-mon[25975]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 217 KiB/s rd, 4.5 KiB/s wr, 389 op/s 2024-06-21T00:44:07.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: mgrmap e20: y(active, since 2m), standbys: x 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 3.7 KiB/s wr, 321 op/s 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:07 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: mgrmap e20: y(active, since 2m), standbys: x 2024-06-21T00:44:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 3.7 KiB/s wr, 321 op/s 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: mgrmap e20: y(active, since 2m), standbys: x 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 3.7 KiB/s wr, 321 op/s 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:07.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:07 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:08.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-06-21T00:44:08.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-06-21T00:44:08.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-06-21T00:44:08.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-06-21T00:44:08.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: osdmap e79: 8 total, 8 up, 8 in 2024-06-21T00:44:08.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1081390398' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:44:08.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:08 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]: dispatch 2024-06-21T00:44:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:08] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-21T00:44:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-06-21T00:44:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-06-21T00:44:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: osdmap e79: 8 total, 8 up, 8 in 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1081390398' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]: dispatch 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: osdmap e79: 8 total, 8 up, 8 in 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1081390398' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:44:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:08 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]: dispatch 2024-06-21T00:44:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:09 smithi120 ceph-mon[25975]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 383 B/s wr, 228 op/s 2024-06-21T00:44:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:09 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]': finished 2024-06-21T00:44:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:09 smithi120 ceph-mon[25975]: osdmap e80: 8 total, 8 up, 8 in 2024-06-21T00:44:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:09 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2798389987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:09 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:09.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[29814]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 383 B/s wr, 228 op/s 2024-06-21T00:44:09.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]': finished 2024-06-21T00:44:09.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[29814]: osdmap e80: 8 total, 8 up, 8 in 2024-06-21T00:44:09.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2798389987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:09.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:09.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[25136]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 383 B/s wr, 228 op/s 2024-06-21T00:44:09.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3437585363' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/4189965795"}]': finished 2024-06-21T00:44:09.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[25136]: osdmap e80: 8 total, 8 up, 8 in 2024-06-21T00:44:09.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2798389987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:09.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:09 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]: dispatch 2024-06-21T00:44:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:10 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]': finished 2024-06-21T00:44:10.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:10 smithi120 ceph-mon[25975]: osdmap e81: 8 total, 8 up, 8 in 2024-06-21T00:44:10.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:10 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]: dispatch 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]': finished 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[25136]: osdmap e81: 8 total, 8 up, 8 in 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]: dispatch 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1048506719"}]': finished 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[29814]: osdmap e81: 8 total, 8 up, 8 in 2024-06-21T00:44:10.948 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:10 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]: dispatch 2024-06-21T00:44:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 341 B/s wr, 34 op/s 2024-06-21T00:44:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]': finished 2024-06-21T00:44:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: osdmap e82: 8 total, 8 up, 8 in 2024-06-21T00:44:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3057229357' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:11.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 341 B/s wr, 34 op/s 2024-06-21T00:44:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]': finished 2024-06-21T00:44:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: osdmap e82: 8 total, 8 up, 8 in 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3057229357' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 341 B/s wr, 34 op/s 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1066211995' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/917100846"}]': finished 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: osdmap e82: 8 total, 8 up, 8 in 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3057229357' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]: dispatch 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:11.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:11.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:44:11.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:44:11.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:44:11.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:44:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]': finished 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: osdmap e83: 8 total, 8 up, 8 in 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:12.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:12 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]: dispatch 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]': finished 2024-06-21T00:44:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: osdmap e83: 8 total, 8 up, 8 in 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]: dispatch 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: Checking dashboard <-> RGW credentials 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3645089549"}]': finished 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: osdmap e83: 8 total, 8 up, 8 in 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:44:12.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:12 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]: dispatch 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[29814]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]': finished 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[29814]: osdmap e84: 8 total, 8 up, 8 in 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]: dispatch 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[25136]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]': finished 2024-06-21T00:44:13.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[25136]: osdmap e84: 8 total, 8 up, 8 in 2024-06-21T00:44:13.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]: dispatch 2024-06-21T00:44:13.981 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:44:13.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:44:13.981 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:44:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:44:13.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:44:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:13 smithi120 ceph-mon[25975]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:13 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3082646984' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/917100846"}]': finished 2024-06-21T00:44:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:13 smithi120 ceph-mon[25975]: osdmap e84: 8 total, 8 up, 8 in 2024-06-21T00:44:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:13 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]: dispatch 2024-06-21T00:44:14.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]': finished 2024-06-21T00:44:14.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[25136]: osdmap e85: 8 total, 8 up, 8 in 2024-06-21T00:44:14.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]: dispatch 2024-06-21T00:44:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]': finished 2024-06-21T00:44:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[29814]: osdmap e85: 8 total, 8 up, 8 in 2024-06-21T00:44:14.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:14 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]: dispatch 2024-06-21T00:44:15.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:14 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/775067515' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1623484127"}]': finished 2024-06-21T00:44:15.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:14 smithi120 ceph-mon[25975]: osdmap e85: 8 total, 8 up, 8 in 2024-06-21T00:44:15.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:14 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]: dispatch 2024-06-21T00:44:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[25136]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]': finished 2024-06-21T00:44:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[25136]: osdmap e86: 8 total, 8 up, 8 in 2024-06-21T00:44:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]: dispatch 2024-06-21T00:44:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[29814]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]': finished 2024-06-21T00:44:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[29814]: osdmap e86: 8 total, 8 up, 8 in 2024-06-21T00:44:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:15 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]: dispatch 2024-06-21T00:44:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:15 smithi120 ceph-mon[25975]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 255 B/s wr, 11 op/s; 13 B/s, 0 objects/s recovering 2024-06-21T00:44:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:15 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3427965987' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1225645423"}]': finished 2024-06-21T00:44:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:15 smithi120 ceph-mon[25975]: osdmap e86: 8 total, 8 up, 8 in 2024-06-21T00:44:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:15 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]: dispatch 2024-06-21T00:44:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]': finished 2024-06-21T00:44:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[29814]: osdmap e87: 8 total, 8 up, 8 in 2024-06-21T00:44:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1500061733' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[29814]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:44:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]': finished 2024-06-21T00:44:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[25136]: osdmap e87: 8 total, 8 up, 8 in 2024-06-21T00:44:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1500061733' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:16 smithi027 ceph-mon[25136]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:44:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:16 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2497325573' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3207268233"}]': finished 2024-06-21T00:44:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:16 smithi120 ceph-mon[25975]: osdmap e87: 8 total, 8 up, 8 in 2024-06-21T00:44:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:16 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1500061733' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:16 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]: dispatch 2024-06-21T00:44:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:16 smithi120 ceph-mon[25975]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:44:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]': finished 2024-06-21T00:44:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[29814]: osdmap e88: 8 total, 8 up, 8 in 2024-06-21T00:44:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]: dispatch 2024-06-21T00:44:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]': finished 2024-06-21T00:44:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[25136]: osdmap e88: 8 total, 8 up, 8 in 2024-06-21T00:44:17.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:17 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]: dispatch 2024-06-21T00:44:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:17 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/581000993"}]': finished 2024-06-21T00:44:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:17 smithi120 ceph-mon[25975]: osdmap e88: 8 total, 8 up, 8 in 2024-06-21T00:44:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:17 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]: dispatch 2024-06-21T00:44:18.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]': finished 2024-06-21T00:44:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[25136]: osdmap e89: 8 total, 8 up, 8 in 2024-06-21T00:44:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]: dispatch 2024-06-21T00:44:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[25136]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]': finished 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[29814]: osdmap e89: 8 total, 8 up, 8 in 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]: dispatch 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-mon[29814]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:18.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:18] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-21T00:44:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:18 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:18 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3164932815' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/581000993"}]': finished 2024-06-21T00:44:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:18 smithi120 ceph-mon[25975]: osdmap e89: 8 total, 8 up, 8 in 2024-06-21T00:44:19.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:18 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]: dispatch 2024-06-21T00:44:19.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:18 smithi120 ceph-mon[25975]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:19 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]': finished 2024-06-21T00:44:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:19 smithi120 ceph-mon[25975]: osdmap e90: 8 total, 8 up, 8 in 2024-06-21T00:44:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:19 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]: dispatch 2024-06-21T00:44:20.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]': finished 2024-06-21T00:44:20.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[25136]: osdmap e90: 8 total, 8 up, 8 in 2024-06-21T00:44:20.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]: dispatch 2024-06-21T00:44:20.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2950073472' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/225303118"}]': finished 2024-06-21T00:44:20.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[29814]: osdmap e90: 8 total, 8 up, 8 in 2024-06-21T00:44:20.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:19 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]: dispatch 2024-06-21T00:44:21.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]': finished 2024-06-21T00:44:21.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:20 smithi120 ceph-mon[25975]: osdmap e91: 8 total, 8 up, 8 in 2024-06-21T00:44:21.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]: dispatch 2024-06-21T00:44:21.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:20 smithi120 ceph-mon[25975]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:21.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]': finished 2024-06-21T00:44:21.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[25136]: osdmap e91: 8 total, 8 up, 8 in 2024-06-21T00:44:21.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]: dispatch 2024-06-21T00:44:21.131 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[25136]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:21.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1806212744' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/225303118"}]': finished 2024-06-21T00:44:21.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[29814]: osdmap e91: 8 total, 8 up, 8 in 2024-06-21T00:44:21.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]: dispatch 2024-06-21T00:44:21.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:20 smithi027 ceph-mon[29814]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:22.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:21 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]': finished 2024-06-21T00:44:22.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:21 smithi120 ceph-mon[25975]: osdmap e92: 8 total, 8 up, 8 in 2024-06-21T00:44:22.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:21 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]: dispatch 2024-06-21T00:44:22.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]': finished 2024-06-21T00:44:22.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[25136]: osdmap e92: 8 total, 8 up, 8 in 2024-06-21T00:44:22.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]: dispatch 2024-06-21T00:44:22.141 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/700248138' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3774793823"}]': finished 2024-06-21T00:44:22.141 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[29814]: osdmap e92: 8 total, 8 up, 8 in 2024-06-21T00:44:22.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:21 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]: dispatch 2024-06-21T00:44:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]': finished 2024-06-21T00:44:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[25136]: osdmap e93: 8 total, 8 up, 8 in 2024-06-21T00:44:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]: dispatch 2024-06-21T00:44:22.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[25136]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]': finished 2024-06-21T00:44:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[29814]: osdmap e93: 8 total, 8 up, 8 in 2024-06-21T00:44:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]: dispatch 2024-06-21T00:44:22.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:22 smithi027 ceph-mon[29814]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:22 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4022947369' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/173316124"}]': finished 2024-06-21T00:44:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:22 smithi120 ceph-mon[25975]: osdmap e93: 8 total, 8 up, 8 in 2024-06-21T00:44:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:22 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]: dispatch 2024-06-21T00:44:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:22 smithi120 ceph-mon[25975]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:23.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:44:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:44:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:23 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]': finished 2024-06-21T00:44:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:23 smithi120 ceph-mon[25975]: osdmap e94: 8 total, 8 up, 8 in 2024-06-21T00:44:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:23 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]: dispatch 2024-06-21T00:44:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]': finished 2024-06-21T00:44:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[25136]: osdmap e94: 8 total, 8 up, 8 in 2024-06-21T00:44:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]: dispatch 2024-06-21T00:44:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1343805545' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2256275255"}]': finished 2024-06-21T00:44:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[29814]: osdmap e94: 8 total, 8 up, 8 in 2024-06-21T00:44:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:23 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]: dispatch 2024-06-21T00:44:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:24 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]': finished 2024-06-21T00:44:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:24 smithi120 ceph-mon[25975]: osdmap e95: 8 total, 8 up, 8 in 2024-06-21T00:44:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:24 smithi120 ceph-mon[25975]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]': finished 2024-06-21T00:44:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[29814]: osdmap e95: 8 total, 8 up, 8 in 2024-06-21T00:44:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[29814]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2448230622' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/665992674"}]': finished 2024-06-21T00:44:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[25136]: osdmap e95: 8 total, 8 up, 8 in 2024-06-21T00:44:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:24 smithi027 ceph-mon[25136]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:27 smithi027 ceph-mon[29814]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-06-21T00:44:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:27 smithi027 ceph-mon[25136]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-06-21T00:44:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:27 smithi120 ceph-mon[25975]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 874 B/s rd, 0 op/s 2024-06-21T00:44:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:28] "GET /metrics HTTP/1.1" 200 208154 "" "Prometheus/2.33.4" 2024-06-21T00:44:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:29 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:29 smithi027 ceph-mon[29814]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T00:44:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:29 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:29 smithi027 ceph-mon[25136]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T00:44:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:29 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:29 smithi120 ceph-mon[25975]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T00:44:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:31 smithi120 ceph-mon[25975]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:44:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:31 smithi027 ceph-mon[25136]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:44:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:31 smithi027 ceph-mon[29814]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:44:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:33 smithi027 ceph-mon[25136]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T00:44:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:33 smithi027 ceph-mon[29814]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T00:44:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:33 smithi120 ceph-mon[25975]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T00:44:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:35 smithi027 ceph-mon[25136]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 946 B/s rd, 0 op/s 2024-06-21T00:44:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:35 smithi027 ceph-mon[29814]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 946 B/s rd, 0 op/s 2024-06-21T00:44:36.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:35 smithi120 ceph-mon[25975]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 946 B/s rd, 0 op/s 2024-06-21T00:44:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:37 smithi027 ceph-mon[25136]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:37 smithi027 ceph-mon[29814]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:37 smithi120 ceph-mon[25975]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:38.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:38] "GET /metrics HTTP/1.1" 200 208109 "" "Prometheus/2.33.4" 2024-06-21T00:44:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:39 smithi027 ceph-mon[25136]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:39.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:39 smithi027 ceph-mon[29814]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:39 smithi120 ceph-mon[25975]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:41.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:41 smithi120 ceph-mon[25975]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:41 smithi027 ceph-mon[25136]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:41 smithi027 ceph-mon[29814]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:43 smithi027 ceph-mon[25136]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:43 smithi027 ceph-mon[29814]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:43 smithi120 ceph-mon[25975]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:45 smithi027 ceph-mon[25136]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:45 smithi027 ceph-mon[29814]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:45 smithi120 ceph-mon[25975]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:47.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:47 smithi027 ceph-mon[25136]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:47 smithi027 ceph-mon[29814]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:47 smithi120 ceph-mon[25975]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:48] "GET /metrics HTTP/1.1" 200 208109 "" "Prometheus/2.33.4" 2024-06-21T00:44:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:49 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:49 smithi027 ceph-mon[25136]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:49 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:49 smithi027 ceph-mon[29814]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:49 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:49 smithi120 ceph-mon[25975]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:51.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:51 smithi120 ceph-mon[25975]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:51 smithi027 ceph-mon[25136]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:51 smithi027 ceph-mon[29814]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:53 smithi027 ceph-mon[25136]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:53 smithi027 ceph-mon[29814]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:53 smithi120 ceph-mon[25975]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:55 smithi027 ceph-mon[25136]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:55 smithi027 ceph-mon[29814]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:55 smithi120 ceph-mon[25975]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:57 smithi027 ceph-mon[25136]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:57 smithi027 ceph-mon[29814]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:57 smithi120 ceph-mon[25975]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:44:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:44:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:44:58.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:44:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:44:58] "GET /metrics HTTP/1.1" 200 208082 "" "Prometheus/2.33.4" 2024-06-21T00:44:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:44:59 smithi027 ceph-mon[25136]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:44:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:44:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:44:59 smithi027 ceph-mon[29814]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:44:59 smithi120 ceph-mon[25975]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:01 smithi120 ceph-mon[25975]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:01 smithi027 ceph-mon[25136]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:01 smithi027 ceph-mon[29814]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:03 smithi027 ceph-mon[25136]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:03.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:03 smithi027 ceph-mon[29814]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:03 smithi120 ceph-mon[25975]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:05.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:05 smithi027 ceph-mon[25136]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:05 smithi027 ceph-mon[29814]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:05 smithi120 ceph-mon[25975]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:45:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:45:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:07 smithi027 ceph-mon[25136]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:07 smithi027 ceph-mon[29814]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:07 smithi120 ceph-mon[25975]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:08] "GET /metrics HTTP/1.1" 200 208043 "" "Prometheus/2.33.4" 2024-06-21T00:45:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:09 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:09 smithi027 ceph-mon[25136]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:09 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:09 smithi027 ceph-mon[29814]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:09 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:09 smithi120 ceph-mon[25975]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:11 smithi120 ceph-mon[25975]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:45:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:45:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:45:11.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:11 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:45:11.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[25136]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[29814]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:45:11.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:11 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:45:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:13 smithi027 ceph-mon[25136]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:13 smithi027 ceph-mon[29814]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:13 smithi120 ceph-mon[25975]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:15 smithi027 ceph-mon[25136]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:15 smithi027 ceph-mon[29814]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:15 smithi120 ceph-mon[25975]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:17 smithi027 ceph-mon[25136]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:17 smithi027 ceph-mon[29814]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:17 smithi120 ceph-mon[25975]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:18.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:18] "GET /metrics HTTP/1.1" 200 208043 "" "Prometheus/2.33.4" 2024-06-21T00:45:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:19 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:19 smithi027 ceph-mon[25136]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:19.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:19 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:19.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:19 smithi027 ceph-mon[29814]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:19 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:19 smithi120 ceph-mon[25975]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:21.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:21 smithi120 ceph-mon[25975]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:21.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:21 smithi027 ceph-mon[25136]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:21 smithi027 ceph-mon[29814]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:23 smithi027 ceph-mon[25136]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:23 smithi027 ceph-mon[29814]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:24.039 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:23 smithi120 ceph-mon[25975]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:25 smithi027 ceph-mon[25136]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:25 smithi027 ceph-mon[29814]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:25 smithi120 ceph-mon[25975]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:27 smithi027 ceph-mon[25136]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:27 smithi027 ceph-mon[29814]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:27 smithi120 ceph-mon[25975]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:28] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-06-21T00:45:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:29 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:29 smithi027 ceph-mon[25136]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:29.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:29 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:29.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:29 smithi027 ceph-mon[29814]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:29 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:29 smithi120 ceph-mon[25975]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:31 smithi120 ceph-mon[25975]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:31 smithi027 ceph-mon[25136]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:31 smithi027 ceph-mon[29814]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:33 smithi120 ceph-mon[25975]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:33 smithi027 ceph-mon[25136]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:33 smithi027 ceph-mon[29814]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:35 smithi027 ceph-mon[25136]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:35 smithi027 ceph-mon[29814]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:36.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:35 smithi120 ceph-mon[25975]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:37 smithi027 ceph-mon[25136]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:37 smithi027 ceph-mon[29814]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:37 smithi120 ceph-mon[25975]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:38] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-21T00:45:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:39 smithi027 ceph-mon[25136]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:39 smithi027 ceph-mon[29814]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:39 smithi120 ceph-mon[25975]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:41.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:41 smithi120 ceph-mon[25975]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:41 smithi027 ceph-mon[25136]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:41 smithi027 ceph-mon[29814]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:43 smithi027 ceph-mon[25136]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:43 smithi027 ceph-mon[29814]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:43 smithi120 ceph-mon[25975]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:45 smithi027 ceph-mon[29814]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:45 smithi027 ceph-mon[25136]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:45 smithi120 ceph-mon[25975]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:47.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:47 smithi027 ceph-mon[25136]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:47 smithi027 ceph-mon[29814]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:47 smithi120 ceph-mon[25975]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:48] "GET /metrics HTTP/1.1" 200 208050 "" "Prometheus/2.33.4" 2024-06-21T00:45:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:49 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:49 smithi027 ceph-mon[25136]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:49 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:49 smithi027 ceph-mon[29814]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:49 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:49 smithi120 ceph-mon[25975]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:51.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:51 smithi120 ceph-mon[25975]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:51 smithi027 ceph-mon[25136]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:51 smithi027 ceph-mon[29814]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:53 smithi027 ceph-mon[25136]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:53 smithi027 ceph-mon[29814]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:53 smithi120 ceph-mon[25975]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:55 smithi027 ceph-mon[25136]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:55 smithi027 ceph-mon[29814]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:55 smithi120 ceph-mon[25975]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:57 smithi027 ceph-mon[25136]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:57 smithi027 ceph-mon[29814]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:57 smithi120 ceph-mon[25975]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:45:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:45:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:45:58.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:45:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:45:58] "GET /metrics HTTP/1.1" 200 208065 "" "Prometheus/2.33.4" 2024-06-21T00:45:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:45:59 smithi027 ceph-mon[25136]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:45:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:45:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:45:59 smithi027 ceph-mon[29814]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:45:59 smithi120 ceph-mon[25975]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:01 smithi120 ceph-mon[25975]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:01 smithi027 ceph-mon[25136]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:01 smithi027 ceph-mon[29814]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:03 smithi027 ceph-mon[25136]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:03.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:03 smithi027 ceph-mon[29814]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:03 smithi120 ceph-mon[25975]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:05.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:05 smithi027 ceph-mon[25136]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:05 smithi027 ceph-mon[29814]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:05 smithi120 ceph-mon[25975]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:46:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:46:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:07 smithi027 ceph-mon[25136]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:07 smithi027 ceph-mon[29814]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:07 smithi120 ceph-mon[25975]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:08] "GET /metrics HTTP/1.1" 200 208058 "" "Prometheus/2.33.4" 2024-06-21T00:46:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:09 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:09 smithi027 ceph-mon[25136]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:09 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:09 smithi027 ceph-mon[29814]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:09 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:09 smithi120 ceph-mon[25975]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:11.843 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:11 smithi120 ceph-mon[25975]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:11.843 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:46:11.844 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:46:11.844 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:11 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[25136]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[29814]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:46:11.854 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:11 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:46:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:12 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:46:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:12 smithi120 ceph-mon[25975]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:12 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:46:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:12 smithi027 ceph-mon[25136]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:12 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:46:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:12 smithi027 ceph-mon[29814]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:15 smithi027 ceph-mon[25136]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:15 smithi027 ceph-mon[29814]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:15 smithi120 ceph-mon[25975]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:17 smithi027 ceph-mon[25136]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:17 smithi027 ceph-mon[29814]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:17 smithi120 ceph-mon[25975]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-06-21T00:46:18.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:18] "GET /metrics HTTP/1.1" 200 208058 "" "Prometheus/2.33.4" 2024-06-21T00:46:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:19 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:19 smithi027 ceph-mon[25136]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:19.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:19 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:19.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:19 smithi027 ceph-mon[29814]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:19 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:19 smithi120 ceph-mon[25975]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:21.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:21 smithi120 ceph-mon[25975]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:21.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:21 smithi027 ceph-mon[25136]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:21 smithi027 ceph-mon[29814]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:23 smithi027 ceph-mon[25136]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:23 smithi027 ceph-mon[29814]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:23 smithi120 ceph-mon[25975]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:25 smithi027 ceph-mon[25136]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:25 smithi027 ceph-mon[29814]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:25 smithi120 ceph-mon[25975]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:27 smithi027 ceph-mon[25136]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:27 smithi027 ceph-mon[29814]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:27 smithi120 ceph-mon[25975]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:28] "GET /metrics HTTP/1.1" 200 208042 "" "Prometheus/2.33.4" 2024-06-21T00:46:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:29 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:29 smithi027 ceph-mon[29814]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:29 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:29.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:29 smithi027 ceph-mon[25136]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:29 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:29 smithi120 ceph-mon[25975]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:31.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:31 smithi120 ceph-mon[25975]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:31 smithi027 ceph-mon[25136]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:31 smithi027 ceph-mon[29814]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:33 smithi027 ceph-mon[25136]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:33 smithi027 ceph-mon[29814]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:33 smithi120 ceph-mon[25975]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:35 smithi027 ceph-mon[25136]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:35 smithi027 ceph-mon[29814]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:36.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:35 smithi120 ceph-mon[25975]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:37 smithi027 ceph-mon[29814]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:37 smithi027 ceph-mon[25136]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:37 smithi120 ceph-mon[25975]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:38] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-06-21T00:46:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:39 smithi027 ceph-mon[25136]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:39 smithi027 ceph-mon[29814]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:39 smithi120 ceph-mon[25975]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:41.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:41 smithi120 ceph-mon[25975]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:41 smithi027 ceph-mon[25136]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:41 smithi027 ceph-mon[29814]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:43 smithi027 ceph-mon[25136]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:43 smithi027 ceph-mon[29814]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:43 smithi120 ceph-mon[25975]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:45 smithi027 ceph-mon[25136]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:45 smithi027 ceph-mon[29814]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:45 smithi120 ceph-mon[25975]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:47.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:47 smithi027 ceph-mon[25136]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:47 smithi027 ceph-mon[29814]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:47 smithi120 ceph-mon[25975]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:48] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-06-21T00:46:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:49 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:49 smithi027 ceph-mon[25136]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:49 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:49 smithi027 ceph-mon[29814]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:49 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:49 smithi120 ceph-mon[25975]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:51.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:51 smithi120 ceph-mon[25975]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:51 smithi027 ceph-mon[25136]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:51 smithi027 ceph-mon[29814]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:53 smithi027 ceph-mon[25136]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:53 smithi027 ceph-mon[29814]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:53 smithi120 ceph-mon[25975]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:55 smithi027 ceph-mon[25136]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:55 smithi027 ceph-mon[29814]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:55 smithi120 ceph-mon[25975]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:57 smithi027 ceph-mon[25136]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:57 smithi027 ceph-mon[29814]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:58.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:57 smithi120 ceph-mon[25975]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:46:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:46:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:46:58.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:46:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:46:58] "GET /metrics HTTP/1.1" 200 208002 "" "Prometheus/2.33.4" 2024-06-21T00:46:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:46:59 smithi027 ceph-mon[25136]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:46:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:46:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:46:59 smithi027 ceph-mon[29814]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:46:59 smithi120 ceph-mon[25975]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:01 smithi027 ceph-mon[25136]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:01 smithi027 ceph-mon[29814]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:01 smithi120 ceph-mon[25975]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:03 smithi027 ceph-mon[25136]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:03.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:03 smithi027 ceph-mon[29814]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:03 smithi120 ceph-mon[25975]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:04.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-06-21T00:47:05.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:05 smithi027 ceph-mon[25136]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:05 smithi027 ceph-mon[29814]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:05 smithi120 ceph-mon[25975]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:06.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:06.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:47:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:47:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:07 smithi027 ceph-mon[25136]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:07 smithi027 ceph-mon[29814]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:07 smithi120 ceph-mon[25975]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:08] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-06-21T00:47:09.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph config set global log_to_journald false --force' 2024-06-21T00:47:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:09 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:09 smithi027 ceph-mon[25136]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:09 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:09 smithi027 ceph-mon[29814]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:09 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:09 smithi120 ceph-mon[25975]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:11.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:11 smithi027 ceph-mon[25136]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:11.892 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:11 smithi027 ceph-mon[29814]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:11.897 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:11 smithi120 ceph-mon[25975]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:12.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T00:47:12.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:12.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:12.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:12 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:12 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:13.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:12 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:13.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:12 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:13.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:12 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:13.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:13 smithi027 ceph-mon[25136]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:13.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:13 smithi027 ceph-mon[29814]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:13 smithi120 ceph-mon[25975]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:14.219 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T00:47:14.219 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c aa8c1dea32b8 2024-06-21T00:47:14.219 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (3m) 3m ago 3m 43.9M - 8.3.5 dad864ee21e9 da2526921115 2024-06-21T00:47:14.219 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (3m) 3m ago 3m 78.0M - 3.5 e1d6a67b021e ea60b303e89f 2024-06-21T00:47:14.219 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443 running (8m) 3m ago 8m 409M - 17.2.0 e1d6a67b021e 8290c77de9e0 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 53e9dc04bcf4 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (8m) 3m ago 8m 39.3M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 3m ago 9m 42.9M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 3m ago 4m 14.2M - 1dbe0e931976 7a46e91c0993 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (4m) 3m ago 4m 14.5M - 1dbe0e931976 7c4e32f60c74 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (8m) 3m ago 8m 48.3M 4016M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 3m ago 7m 51.3M 4016M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (7m) 3m ago 7m 43.4M 4016M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (7m) 3m ago 7m 46.2M 4016M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (6m) 3m ago 6m 47.1M 4272M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (6m) 3m ago 6m 45.6M 4272M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T00:47:14.220 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (5m) 3m ago 5m 44.1M 4272M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T00:47:14.221 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (5m) 3m ago 5m 46.8M 4272M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T00:47:14.221 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (3m) 3m ago 4m 43.7M - 514e6a882f6e d65c4d07f53c 2024-06-21T00:47:14.221 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (3m) 3m ago 3m 72.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T00:47:14.221 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (3m) 3m ago 3m 68.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T00:47:14.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T00:47:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:15 smithi027 ceph-mon[29814]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:15 smithi027 ceph-mon[29814]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:15 smithi027 ceph-mon[25136]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:15 smithi027 ceph-mon[25136]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:15 smithi120 ceph-mon[25975]: from='client.14961 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:15 smithi120 ceph-mon[25975]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:16.855 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T00:47:16.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:47:16.857 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T00:47:16.857 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-06-21T00:47:16.857 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:47:16.857 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:47:17.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph -s' 2024-06-21T00:47:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:17 smithi027 ceph-mon[25136]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:17 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4220150477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:47:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:17 smithi027 ceph-mon[29814]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:17 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4220150477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:47:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:17 smithi120 ceph-mon[25975]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:17 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4220150477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:47:18.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:18] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: id: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 5m), standbys: x 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-06-21T00:47:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 241 objects, 585 KiB 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-06-21T00:47:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:47:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:19 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:19 smithi027 ceph-mon[25136]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:19.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:19 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:19.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:19 smithi027 ceph-mon[29814]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:19 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:19 smithi120 ceph-mon[25975]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:20.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ls' 2024-06-21T00:47:20.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1733468854' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:47:20.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1733468854' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:47:21.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1733468854' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:47:21.805 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:21 smithi027 ceph-mon[29814]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:21.805 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:21 smithi027 ceph-mon[25136]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:22.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:21 smithi120 ceph-mon[25975]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:22.488 INFO:teuthology.orchestra.run.smithi027.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-21T00:47:22.488 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi027=a;count:1 2024-06-21T00:47:22.488 INFO:teuthology.orchestra.run.smithi027.stdout:grafana ?:3000 1/1 3m ago 5m smithi120=a;count:1 2024-06-21T00:47:22.488 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:mgr 2/2 3m ago 8m smithi027=y;smithi120=x;count:2 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:mon 3/3 3m ago 9m smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b;count:3 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi027=a;smithi120=b;count:2 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:osd 8 3m ago - 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus ?:9095 1/1 3m ago 5m smithi120=a;count:1 2024-06-21T00:47:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-06-21T00:47:23.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-21T00:47:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:23 smithi027 ceph-mon[29814]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:23 smithi027 ceph-mon[29814]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:23 smithi027 ceph-mon[25136]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:23.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:23 smithi027 ceph-mon[25136]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:23 smithi120 ceph-mon[25975]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:23 smithi120 ceph-mon[25975]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:25.544 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled to redeploy mgr.x on host 'smithi120' 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[25136]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1639764088' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[29814]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:25.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1639764088' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:47:25.905 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:25.905 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:25 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:25 smithi120 ceph-mon[25975]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:25 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1639764088' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:47:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:25 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:25 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:47:26.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps --refresh' 2024-06-21T00:47:26.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:26 smithi027 ceph-mon[25136]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:26.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:26 smithi027 ceph-mon[25136]: Schedule redeploy daemon mgr.x 2024-06-21T00:47:26.859 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:26 smithi027 ceph-mon[29814]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:26.859 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:26 smithi027 ceph-mon[29814]: Schedule redeploy daemon mgr.x 2024-06-21T00:47:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:26 smithi120 ceph-mon[25975]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:26 smithi120 ceph-mon[25975]: Schedule redeploy daemon mgr.x 2024-06-21T00:47:27.854 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:27 smithi027 ceph-mon[25136]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:27.854 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:27 smithi027 ceph-mon[29814]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:27 smithi120 ceph-mon[25975]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:28.602 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T00:47:28.602 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c aa8c1dea32b8 2024-06-21T00:47:28.602 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (4m) 3m ago 4m 43.9M - 8.3.5 dad864ee21e9 da2526921115 2024-06-21T00:47:28.602 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (3m) 3m ago 3m 78.0M - 3.5 e1d6a67b021e ea60b303e89f 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443 running (8m) 3m ago 8m 409M - 17.2.0 e1d6a67b021e 8290c77de9e0 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 53e9dc04bcf4 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (9m) 3m ago 9m 39.3M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 3m ago 9m 42.9M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (5m) 3m ago 5m 14.2M - 1dbe0e931976 7a46e91c0993 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (4m) 3m ago 4m 14.5M - 1dbe0e931976 7c4e32f60c74 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (8m) 3m ago 8m 48.3M 4016M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (8m) 3m ago 8m 51.3M 4016M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (7m) 3m ago 7m 43.4M 4016M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (7m) 3m ago 7m 46.2M 4016M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (6m) 3m ago 6m 47.1M 4272M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T00:47:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (6m) 3m ago 6m 45.6M 4272M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T00:47:28.604 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (6m) 3m ago 6m 44.1M 4272M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T00:47:28.604 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (5m) 3m ago 5m 46.8M 4272M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T00:47:28.604 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (3m) 3m ago 5m 43.7M - 514e6a882f6e d65c4d07f53c 2024-06-21T00:47:28.604 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (3m) 3m ago 3m 72.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T00:47:28.604 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (3m) 3m ago 3m 68.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T00:47:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[25136]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[25136]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:28] "GET /metrics HTTP/1.1" 200 207997 "" "Prometheus/2.33.4" 2024-06-21T00:47:28.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:28.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[29814]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:28.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:28 smithi027 ceph-mon[29814]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:28 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:28 smithi120 ceph-mon[25975]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:47:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:28 smithi120 ceph-mon[25975]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:30.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T00:47:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:31 smithi027 ceph-mon[25136]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:31 smithi027 ceph-mon[29814]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:32.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:31 smithi120 ceph-mon[25975]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:33 smithi027 ceph-mon[25136]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:33 smithi027 ceph-mon[29814]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:33 smithi120 ceph-mon[25975]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:35 smithi027 ceph-mon[25136]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:35 smithi027 ceph-mon[29814]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:36.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:35 smithi120 ceph-mon[25975]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:37.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:37 smithi027 ceph-mon[25136]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:37 smithi027 ceph-mon[29814]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:37 smithi120 ceph-mon[25975]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:38] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-06-21T00:47:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:39 smithi027 ceph-mon[25136]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:39 smithi027 ceph-mon[29814]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:39 smithi120 ceph-mon[25975]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:41 smithi027 ceph-mon[25136]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:41 smithi027 ceph-mon[29814]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:41 smithi120 ceph-mon[25975]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:43 smithi027 ceph-mon[25136]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:43 smithi027 ceph-mon[29814]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:44.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:43 smithi120 ceph-mon[25975]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:45 smithi027 ceph-mon[25136]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:45 smithi027 ceph-mon[29814]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:45 smithi120 ceph-mon[25975]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:47.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:47 smithi027 ceph-mon[25136]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:47 smithi027 ceph-mon[29814]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:47 smithi120 ceph-mon[25975]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:48] "GET /metrics HTTP/1.1" 200 207994 "" "Prometheus/2.33.4" 2024-06-21T00:47:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:49 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:49 smithi027 ceph-mon[25136]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:49 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:49 smithi027 ceph-mon[29814]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:49 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:49 smithi120 ceph-mon[25975]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:51.919 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:51 smithi120 ceph-mon[25975]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:51 smithi027 ceph-mon[25136]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:51 smithi027 ceph-mon[29814]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:53 smithi027 ceph-mon[25136]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:53 smithi027 ceph-mon[29814]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:53 smithi120 ceph-mon[25975]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[25136]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:55.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:55.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[29814]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:55.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:55.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:55.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:55.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:55 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:55 smithi120 ceph-mon[25975]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:55 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:55 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:55 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:47:56.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:55 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:57 smithi027 ceph-mon[25136]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:57 smithi027 ceph-mon[29814]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:58.074 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:57 smithi120 ceph-mon[25975]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:47:58.074 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:47:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:47:58.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:47:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:47:58] "GET /metrics HTTP/1.1" 200 208004 "" "Prometheus/2.33.4" 2024-06-21T00:47:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:47:59.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:59.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:47:59 smithi120 ceph-mon[25975]: Deploying daemon mgr.x on smithi120 2024-06-21T00:47:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[25136]: Deploying daemon mgr.x on smithi120 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:47:59.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:47:59.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:47:59.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:47:59.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:47:59 smithi027 ceph-mon[29814]: Deploying daemon mgr.x on smithi120 2024-06-21T00:48:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:01 smithi027 ceph-mon[25136]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:01.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:01 smithi027 ceph-mon[29814]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:01 smithi120 ceph-mon[25975]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:03.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:03 smithi027 ceph-mon[25136]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:03.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:03 smithi027 ceph-mon[29814]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:04.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:03 smithi120 ceph-mon[25975]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:05.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:05 smithi027 ceph-mon[25136]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:05 smithi027 ceph-mon[29814]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:05 smithi120 ceph-mon[25975]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:48:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:48:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:07 smithi027 ceph-mon[25136]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:07 smithi027 ceph-mon[29814]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:07 smithi120 ceph-mon[25975]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:48:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:08 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:08 smithi027 ceph-mon[25136]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:08.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:08] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-06-21T00:48:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:08 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:08.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:08 smithi027 ceph-mon[29814]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:09.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:08 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:08 smithi120 ceph-mon[25975]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:11 smithi027 ceph-mon[25136]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:11 smithi027 ceph-mon[29814]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:11 smithi120 ceph-mon[25975]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:13 smithi027 ceph-mon[25136]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:13 smithi027 ceph-mon[29814]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:13.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:13 smithi120 ceph-mon[25975]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:15 smithi027 ceph-mon[25136]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:15 smithi027 ceph-mon[29814]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:15 smithi120 ceph-mon[25975]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:17.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:17 smithi027 ceph-mon[25136]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:17.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:17 smithi027 ceph-mon[29814]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:17 smithi120 ceph-mon[25975]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:18.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[27143]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:48:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:18 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:18 smithi027 ceph-mon[25136]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:18 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:18 smithi027 ceph-mon[29814]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:18] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-06-21T00:48:19.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:18 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:18 smithi120 ceph-mon[25975]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:21.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:21 smithi027 ceph-mon[25136]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:21 smithi027 ceph-mon[29814]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:21 smithi120 ceph-mon[25975]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:23 smithi027 ceph-mon[25136]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:23 smithi027 ceph-mon[29814]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:48:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:24.084 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:23 smithi120 ceph-mon[25975]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:25 smithi027 ceph-mon[25136]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:25 smithi027 ceph-mon[29814]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:25.979 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:25 smithi120 ceph-mon[25975]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:26.335 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 systemd[1]: Stopping Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:48:26.335 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 bash[45851]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr.x" found: no such container 2024-06-21T00:48:26.606 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 podman[45861]: 2024-06-21 00:48:26.336562196 +0000 UTC m=+0.118627530 container died 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.component=centos-stream-container, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, version=8, release=754, name=centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-06-21T00:48:26.865 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 podman[45861]: 2024-06-21 00:48:26.665750107 +0000 UTC m=+0.447815436 container cleanup 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, version=8, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., name=centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8) 2024-06-21T00:48:26.865 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 bash[45861]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x 2024-06-21T00:48:26.865 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 bash[45889]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr.x" found: no such container 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 podman[45874]: 2024-06-21 00:48:26.867508565 +0000 UTC m=+0.524209469 container remove 8290c77de9e0d09133818df675ef06d28edf387fea6c267b03bc4a4b03f6afd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, architecture=x86_64) 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 systemd[1]: Stopped Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:26 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Consumed 17.356s CPU time. 2024-06-21T00:48:27.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 systemd[1]: Starting Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:48:27.530 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 podman[45989]: 2024-06-21 00:48:27.184052364 +0000 UTC m=+0.016994433 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T00:48:27.531 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 podman[45989]: 2024-06-21 00:48:27.318825546 +0000 UTC m=+0.151767605 container create 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-06-21T00:48:27.531 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 podman[45989]: 2024-06-21 00:48:27.530087353 +0000 UTC m=+0.363029423 container init 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, ceph=True) 2024-06-21T00:48:27.531 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 podman[45989]: 2024-06-21 00:48:27.53284326 +0000 UTC m=+0.365785321 container start 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=) 2024-06-21T00:48:27.825 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-mgr[46008]: -- 172.21.15.120:0/431892949 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55db5d58f0e0 con 0x55db5d56d000 2024-06-21T00:48:27.826 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 bash[45989]: 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df 2024-06-21T00:48:27.827 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 systemd[1]: Started Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:48:27.827 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:27.659+0000 7f3a39a70040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:48:27.827 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:27.825+0000 7f3a39a70040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:48:27.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:27.680Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": dial tcp 172.21.15.120:8443: connect: connection refused" 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:48:27.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:48:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:27 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:48:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:27.899+0000 7f3a39a70040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:48:28.519 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:28.170+0000 7f3a39a70040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:48:28.520 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:28.238+0000 7f3a39a70040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:48:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:28] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-06-21T00:48:29.266 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:28 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:29.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:28 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:29.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:28 smithi120 ceph-mon[25975]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[25136]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:29.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:28 smithi027 ceph-mon[29814]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:29.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.267+0000 7f3a39a70040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:48:29.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.388+0000 7f3a39a70040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:48:29.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.447+0000 7f3a39a70040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:48:29.981 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.680+0000 7f3a39a70040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:48:29.981 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.751+0000 7f3a39a70040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:48:29.982 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.828+0000 7f3a39a70040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:48:29.982 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:29.982+0000 7f3a39a70040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:48:30.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:30.420+0000 7f3a39a70040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:48:31.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:30.761+0000 7f3a39a70040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:48:31.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:30.834+0000 7f3a39a70040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:48:31.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:30.990+0000 7f3a39a70040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T00:48:31.582 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.324+0000 7f3a39a70040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:48:31.583 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.388+0000 7f3a39a70040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:48:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-mon[25975]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:31.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.584+0000 7f3a39a70040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:48:31.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.741+0000 7f3a39a70040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:48:31.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.807+0000 7f3a39a70040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:48:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:31 smithi027 ceph-mon[25136]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:31 smithi027 ceph-mon[29814]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:32.243 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.894+0000 7f3a39a70040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:48:32.243 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:31.960+0000 7f3a39a70040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:48:32.794 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:32.522+0000 7f3a39a70040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:48:32.795 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:32.686+0000 7f3a39a70040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:48:32.795 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:48:32] ENGINE Bus STARTING 2024-06-21T00:48:32.795 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: CherryPy Checker: 2024-06-21T00:48:32.795 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: The Application mounted at '' has an empty config. 2024-06-21T00:48:32.796 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:48:32.796 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:48:32] ENGINE Serving on http://:::9283 2024-06-21T00:48:32.796 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:48:32] ENGINE Bus STARTED 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: Standby manager daemon x restarted 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:48:33.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:48:33.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:48:33.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:32 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:48:33.229 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:32.977Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: Standby manager daemon x restarted 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:48:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: Standby manager daemon x restarted 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:48:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:32 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:48:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:48:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:34.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:34 smithi120 ceph-mon[25975]: mgrmap e21: y(active, since 6m), standbys: x 2024-06-21T00:48:34.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:34 smithi027 ceph-mon[29814]: mgrmap e21: y(active, since 6m), standbys: x 2024-06-21T00:48:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:34 smithi027 ceph-mon[25136]: mgrmap e21: y(active, since 6m), standbys: x 2024-06-21T00:48:35.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:35 smithi120 ceph-mon[25975]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:35 smithi027 ceph-mon[25136]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:35.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:35 smithi027 ceph-mon[29814]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:37 smithi120 ceph-mon[25975]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:48:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:37 smithi027 ceph-mon[25136]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:37 smithi027 ceph-mon[29814]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:38] "GET /metrics HTTP/1.1" 200 208010 "" "Prometheus/2.33.4" 2024-06-21T00:48:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:39 smithi120 ceph-mon[25975]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:39 smithi027 ceph-mon[25136]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:39 smithi027 ceph-mon[29814]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:41 smithi027 ceph-mon[25136]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:41 smithi027 ceph-mon[29814]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:41 smithi120 ceph-mon[25975]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:43 smithi027 ceph-mon[25136]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:43 smithi027 ceph-mon[29814]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:43.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:48:43.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:43.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:43.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:43 smithi120 ceph-mon[25975]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:45 smithi120 ceph-mon[25975]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:46.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:45 smithi027 ceph-mon[25136]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:46.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:45 smithi027 ceph-mon[29814]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:46 smithi120 ceph-mon[25975]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:47.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:46 smithi027 ceph-mon[25136]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:47.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:46 smithi027 ceph-mon[29814]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:48.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:48:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:48] "GET /metrics HTTP/1.1" 200 208010 "" "Prometheus/2.33.4" 2024-06-21T00:48:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:49 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:49 smithi120 ceph-mon[25975]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:49 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:49 smithi027 ceph-mon[29814]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:50.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:49 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:48:50.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:49 smithi027 ceph-mon[25136]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:51.918 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:51 smithi120 ceph-mon[25975]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:51.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:51 smithi027 ceph-mon[25136]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:51 smithi027 ceph-mon[29814]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:53 smithi027 ceph-mon[25136]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:53 smithi027 ceph-mon[29814]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:53.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:48:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:53.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:48:53.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:48:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:48:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:48:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:53 smithi120 ceph-mon[25975]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:48:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:55 smithi120 ceph-mon[25975]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:56.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:55 smithi027 ceph-mon[25136]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:56.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:55 smithi027 ceph-mon[29814]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:57.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:56 smithi120 ceph-mon[25975]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:56 smithi027 ceph-mon[25136]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:57.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:56 smithi027 ceph-mon[29814]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:48:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:48:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:48:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:48:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:48:58] "GET /metrics HTTP/1.1" 200 208017 "" "Prometheus/2.33.4" 2024-06-21T00:49:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:48:59 smithi120 ceph-mon[25975]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:48:59 smithi027 ceph-mon[25136]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:48:59 smithi027 ceph-mon[29814]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:01 smithi027 ceph-mon[25136]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:01 smithi027 ceph-mon[29814]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:02.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:01 smithi120 ceph-mon[25975]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:03 smithi027 ceph-mon[25136]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:03.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:03 smithi027 ceph-mon[29814]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:03.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:03.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:03.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:03 smithi120 ceph-mon[25975]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:04 smithi120 ceph-mon[25975]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:04 smithi027 ceph-mon[25136]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:05.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:04 smithi027 ceph-mon[29814]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:49:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:49:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:07 smithi120 ceph-mon[25975]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:08.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:07 smithi027 ceph-mon[25136]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:07 smithi027 ceph-mon[29814]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:08.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:08 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:08.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:08 smithi027 ceph-mon[29814]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:08 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:08.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:08 smithi027 ceph-mon[25136]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:08.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:08] "GET /metrics HTTP/1.1" 200 208018 "" "Prometheus/2.33.4" 2024-06-21T00:49:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:08 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:08 smithi120 ceph-mon[25975]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:11 smithi027 ceph-mon[25136]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:11 smithi027 ceph-mon[29814]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:11 smithi120 ceph-mon[25975]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:13.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:12 smithi120 ceph-mon[25975]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:12 smithi027 ceph-mon[25136]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:12 smithi027 ceph-mon[29814]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:14 smithi120 ceph-mon[25975]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:14 smithi027 ceph-mon[25136]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:14 smithi027 ceph-mon[29814]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:16 smithi120 ceph-mon[25975]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:17.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:16 smithi027 ceph-mon[25136]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:16 smithi027 ceph-mon[29814]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:18.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:18 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:18 smithi027 ceph-mon[29814]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:18 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:18 smithi027 ceph-mon[25136]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:18] "GET /metrics HTTP/1.1" 200 208018 "" "Prometheus/2.33.4" 2024-06-21T00:49:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:18 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:18 smithi120 ceph-mon[25975]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:21.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:21 smithi027 ceph-mon[25136]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:21 smithi027 ceph-mon[29814]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:21 smithi120 ceph-mon[25975]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:22 smithi120 ceph-mon[25975]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:23.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:22 smithi027 ceph-mon[25136]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:22 smithi027 ceph-mon[29814]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:23.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:24 smithi120 ceph-mon[25975]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:25.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:24 smithi027 ceph-mon[25136]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:24 smithi027 ceph-mon[29814]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:26 smithi120 ceph-mon[25975]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:26 smithi027 ceph-mon[25136]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:26 smithi027 ceph-mon[29814]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:28.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:28 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:28.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:28 smithi027 ceph-mon[29814]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:28 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:28 smithi027 ceph-mon[25136]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:28.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:28] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-06-21T00:49:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:28 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:28 smithi120 ceph-mon[25975]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:31 smithi027 ceph-mon[25136]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:31 smithi027 ceph-mon[29814]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:32.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:31 smithi120 ceph-mon[25975]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:32.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:49:32.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:49:32.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:49:32.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:32 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:49:32.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:49:32.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:32 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:49:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:33 smithi027 ceph-mon[25136]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:33 smithi027 ceph-mon[29814]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:33.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:33.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:33 smithi120 ceph-mon[25975]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:35.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:34 smithi120 ceph-mon[25975]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:34 smithi027 ceph-mon[25136]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:34 smithi027 ceph-mon[29814]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:37.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:36 smithi120 ceph-mon[25975]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:37.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:36 smithi027 ceph-mon[25136]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:37.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:36 smithi027 ceph-mon[29814]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:38.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:38 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:38 smithi027 ceph-mon[29814]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:38 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:38 smithi027 ceph-mon[25136]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:38] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-06-21T00:49:39.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:38 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:39.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:38 smithi120 ceph-mon[25975]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:41 smithi027 ceph-mon[25136]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:41 smithi027 ceph-mon[29814]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:41 smithi120 ceph-mon[25975]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:42 smithi120 ceph-mon[25975]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:43.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:42 smithi027 ceph-mon[25136]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:43.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:42 smithi027 ceph-mon[29814]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:43.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:43.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:43.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:44 smithi120 ceph-mon[25975]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:44 smithi027 ceph-mon[29814]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:44 smithi027 ceph-mon[25136]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:47.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:46 smithi120 ceph-mon[25975]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:47.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:46 smithi027 ceph-mon[25136]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:47.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:46 smithi027 ceph-mon[29814]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:48 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:48 smithi027 ceph-mon[25136]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:48] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-06-21T00:49:48.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:48 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:48.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:48 smithi027 ceph-mon[29814]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:48 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:49.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:48 smithi120 ceph-mon[25975]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:51 smithi027 ceph-mon[25136]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:51 smithi027 ceph-mon[29814]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:51 smithi120 ceph-mon[25975]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:52 smithi120 ceph-mon[25975]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:52 smithi027 ceph-mon[25136]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:52 smithi027 ceph-mon[29814]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:53.978 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:49:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:53.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:49:53.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:49:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:49:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:49:55.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:54 smithi120 ceph-mon[25975]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:54 smithi027 ceph-mon[25136]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:54 smithi027 ceph-mon[29814]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:56 smithi120 ceph-mon[25975]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:56 smithi027 ceph-mon[25136]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:57.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:56 smithi027 ceph-mon[29814]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:49:58.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:49:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:49:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:58 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:49:58 smithi027 ceph-mon[25136]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:58 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:49:58 smithi027 ceph-mon[29814]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:49:58.980 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:49:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:49:58] "GET /metrics HTTP/1.1" 200 208017 "" "Prometheus/2.33.4" 2024-06-21T00:49:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:58 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:49:59.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:49:58 smithi120 ceph-mon[25975]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:00.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:00 smithi120 ceph-mon[25975]: overall HEALTH_OK 2024-06-21T00:50:00.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:00 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T00:50:00.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:00 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T00:50:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:01 smithi027 ceph-mon[25136]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:01 smithi027 ceph-mon[29814]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:01 smithi120 ceph-mon[25975]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:03.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:02 smithi120 ceph-mon[25975]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:02 smithi027 ceph-mon[25136]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:02 smithi027 ceph-mon[29814]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:03.981 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:03.982 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:03.982 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:04 smithi120 ceph-mon[25975]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:04 smithi027 ceph-mon[25136]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:05.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:04 smithi027 ceph-mon[29814]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:06.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:06 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:06 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:07.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:06 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:07 smithi027 ceph-mon[25136]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:07 smithi027 ceph-mon[29814]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:08.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:07 smithi120 ceph-mon[25975]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:08] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-06-21T00:50:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:09 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:09 smithi027 ceph-mon[25136]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:09 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:09 smithi027 ceph-mon[29814]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:10.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:09 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:10.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:09 smithi120 ceph-mon[25975]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:11 smithi027 ceph-mon[25136]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:11 smithi027 ceph-mon[29814]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:11 smithi120 ceph-mon[25975]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:12 smithi120 ceph-mon[25975]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:12 smithi027 ceph-mon[25136]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:12 smithi027 ceph-mon[29814]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:13.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:14 smithi120 ceph-mon[25975]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:14 smithi027 ceph-mon[25136]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:14 smithi027 ceph-mon[29814]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:16 smithi120 ceph-mon[25975]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:16 smithi027 ceph-mon[25136]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:16 smithi027 ceph-mon[29814]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:18.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:18.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:18 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:18.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:18 smithi027 ceph-mon[25136]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:18 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:18 smithi027 ceph-mon[29814]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:18] "GET /metrics HTTP/1.1" 200 208019 "" "Prometheus/2.33.4" 2024-06-21T00:50:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:18 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:19.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:18 smithi120 ceph-mon[25975]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:21.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:21 smithi027 ceph-mon[25136]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:21.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:21 smithi027 ceph-mon[29814]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:21 smithi120 ceph-mon[25975]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:22 smithi120 ceph-mon[25975]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:23.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:22 smithi027 ceph-mon[25136]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:23.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:22 smithi027 ceph-mon[29814]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:23.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:23.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:23.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:24 smithi120 ceph-mon[25975]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:25.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:24 smithi027 ceph-mon[25136]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:24 smithi027 ceph-mon[29814]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:27.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:26 smithi120 ceph-mon[25975]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:26 smithi027 ceph-mon[25136]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:26 smithi027 ceph-mon[29814]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:28.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:28.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:28 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:28.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:28 smithi027 ceph-mon[25136]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:28.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:28 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:28.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:28 smithi027 ceph-mon[29814]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:28] "GET /metrics HTTP/1.1" 200 208018 "" "Prometheus/2.33.4" 2024-06-21T00:50:29.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:28 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:29.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:28 smithi120 ceph-mon[25975]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:31 smithi027 ceph-mon[25136]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:31 smithi027 ceph-mon[29814]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:31 smithi120 ceph-mon[25975]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:32.909 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:32.909 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:32.909 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:32 smithi120 ceph-mon[25975]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:32.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:32.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:32.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[25136]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:32.924 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:32.924 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:32.924 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:32 smithi027 ceph-mon[29814]: from='mgr.24343 172.21.15.27:0/918313980' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:33.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T00:50:33.644 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:33.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:33.644 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:33.645 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:33.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-mon[25136]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:33.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-mon[25136]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:50:33.911 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-mon[29814]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:33.911 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:33 smithi027 ceph-mon[29814]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:50:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:33 smithi120 ceph-mon[25975]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:33 smithi120 ceph-mon[25975]: from='mgr.24343 ' entity='mgr.y' 2024-06-21T00:50:34.944 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:34 smithi027 ceph-mon[25136]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:34.944 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:34 smithi027 ceph-mon[29814]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:34 smithi120 ceph-mon[25975]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:35.396 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (6m) 2m ago 7m 25.1M - ba2b418f427c aa8c1dea32b8 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (7m) 2m ago 7m 53.4M - 8.3.5 dad864ee21e9 da2526921115 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (6m) 2m ago 6m 55.4M - 3.5 e1d6a67b021e ea60b303e89f 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (2m) 2m ago 11m 77.1M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e 53e9dc04bcf4 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (13m) 2m ago 13m 50.8M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (12m) 2m ago 12m 39.2M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (12m) 2m ago 12m 40.2M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 2m ago 8m 17.6M - 1dbe0e931976 7a46e91c0993 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (8m) 2m ago 8m 15.2M - 1dbe0e931976 7c4e32f60c74 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (11m) 2m ago 11m 51.3M 4016M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (11m) 2m ago 11m 53.0M 4016M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T00:50:35.397 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (10m) 2m ago 10m 45.8M 4016M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (10m) 2m ago 10m 49.6M 4016M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (10m) 2m ago 10m 50.3M 4272M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (9m) 2m ago 9m 49.4M 4272M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (9m) 2m ago 9m 47.5M 4272M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (8m) 2m ago 8m 49.3M 4272M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (6m) 2m ago 8m 68.3M - 514e6a882f6e d65c4d07f53c 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (6m) 2m ago 6m 73.6M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T00:50:35.398 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (6m) 2m ago 6m 71.4M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T00:50:36.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T00:50:36.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:36 smithi027 ceph-mon[25136]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:50:36.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:36 smithi027 ceph-mon[29814]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:50:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:36 smithi120 ceph-mon[25975]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:50:37.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:37 smithi027 ceph-mon[25136]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:37 smithi027 ceph-mon[29814]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:37 smithi120 ceph-mon[25975]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:38.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T00:50:38.167 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 1 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T00:50:38.168 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-06-21T00:50:38.169 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 1 2024-06-21T00:50:38.169 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:50:38.169 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:50:38.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:38 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1765203735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:38.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:38 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1765203735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:38.721 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:38] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-06-21T00:50:38.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph -s' 2024-06-21T00:50:39.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:38 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1765203735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:39 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:39 smithi027 ceph-mon[29814]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:39 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:39 smithi027 ceph-mon[25136]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:39 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:40.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:39 smithi120 ceph-mon[25975]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:40.752 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-06-21T00:50:40.752 INFO:teuthology.orchestra.run.smithi027.stdout: id: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 8m), standbys: x 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 241 objects, 585 KiB 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:40.753 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-06-21T00:50:40.754 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:50:40.754 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-06-21T00:50:40.754 INFO:teuthology.orchestra.run.smithi027.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-06-21T00:50:40.754 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:50:41.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T00:50:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:41 smithi027 ceph-mon[25136]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:41 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4008989191' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:50:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:41 smithi027 ceph-mon[29814]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:41 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4008989191' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:50:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:41 smithi120 ceph-mon[25975]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:41 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4008989191' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:50:42.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:42 smithi027 ceph-mon[25136]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:42.960 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:42 smithi027 ceph-mon[29814]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:42 smithi120 ceph-mon[25975]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:43.440 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T00:50:43.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:43.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:43.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:43.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:43 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:44.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:43 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2463265081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:50:44.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:43 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2463265081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:50:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:43 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2463265081' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:50:44.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-06-21T00:50:45.112 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:44 smithi027 ceph-mon[25136]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:45.113 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:44 smithi027 ceph-mon[29814]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:44 smithi120 ceph-mon[25975]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:46.140 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T00:50:46.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:46 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3333440755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:46.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:46 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3333440755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:46.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph mgr fail' 2024-06-21T00:50:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:46 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3333440755' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:50:47.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:47 smithi027 ceph-mon[25136]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:47.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:47 smithi027 ceph-mon[29814]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:47 smithi120 ceph-mon[25975]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:50:48.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:47 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:48.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:48 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:48 smithi027 ceph-mon[25136]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:48.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:48 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:48.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:48 smithi027 ceph-mon[29814]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:48] "GET /metrics HTTP/1.1" 200 208014 "" "Prometheus/2.33.4" 2024-06-21T00:50:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:48 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:49.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:48 smithi120 ceph-mon[25975]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:50:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:49 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2971576681' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:49 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:49 smithi120 ceph-mon[25975]: osdmap e96: 8 total, 8 up, 8 in 2024-06-21T00:50:50.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:49 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:49] ENGINE Bus STOPPING 2024-06-21T00:50:50.134 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2971576681' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.134 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.134 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[25136]: osdmap e96: 8 total, 8 up, 8 in 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2971576681' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mon[29814]: osdmap e96: 8 total, 8 up, 8 in 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:49.816+0000 7f6e58fc9700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ignoring --setgroup ceph since I am not root 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:49 smithi027 ceph-mgr[25369]: -- 172.21.15.27:0/4194240237 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bb96236340 con 0x55bb9620f000 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:50.002+0000 7f7eca0fb000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:50:50.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:50.135+0000 7f7eca0fb000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:50:50.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:50.240+0000 7f7eca0fb000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:50:50.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T00:50:50.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:50] ENGINE Bus STOPPED 2024-06-21T00:50:50.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:50] ENGINE Bus STARTING 2024-06-21T00:50:50.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:50] ENGINE Serving on http://:::9283 2024-06-21T00:50:50.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:50:50] ENGINE Bus STARTED 2024-06-21T00:50:50.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T00:50:50.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:50:50.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: mgrmap e22: x(active, starting, since 0.769926s) 2024-06-21T00:50:50.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:50:50.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:50.567+0000 7f7eca0fb000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:50:50.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:50.657+0000 7f7eca0fb000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: mgrmap e22: x(active, starting, since 0.769926s) 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:50:51.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: Manager daemon x is now available 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: Queued rgw.foo for migration 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.094 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: Manager daemon x is now available 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: Queued rgw.foo for migration 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-21T00:50:51.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:50 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: mgrmap e22: x(active, starting, since 0.769926s) 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:50:51.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: Manager daemon x is now available 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: Queued rgw.foo for migration 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:50 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: mgrmap e23: x(active, since 1.77963s) 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: Deploying cephadm binary to smithi120 2024-06-21T00:50:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[25136]: Deploying cephadm binary to smithi027 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: mgrmap e23: x(active, since 1.77963s) 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: Deploying cephadm binary to smithi120 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-mon[29814]: Deploying cephadm binary to smithi027 2024-06-21T00:50:51.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:51.912+0000 7f7eca0fb000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:50:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: mgrmap e23: x(active, since 1.77963s) 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: Deploying cephadm binary to smithi120 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-mon[25975]: Deploying cephadm binary to smithi027 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:50:51.934+0000 7f3996d2b640 -1 log_channel(cephadm) log [ERR] : [21/Jun/2024:00:50:51] ENGINE Error in HTTPServer.serve 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:50:52.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._connections.run(self.expiration_interval) 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._run(expiration_interval) 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: s = self.context.wrap_socket( 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.sslsocket_class._create( 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:50:52.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.do_handshake() 2024-06-21T00:50:52.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:50:52.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._sslobj.do_handshake() 2024-06-21T00:50:52.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:50:52.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:51 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:50:52.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:52.100+0000 7f7eca0fb000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:50:52.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:52.187+0000 7f7eca0fb000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:50:52.229 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:51 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:51.980Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": dial tcp 172.21.15.27:8443: connect: connection refused" 2024-06-21T00:50:52.831 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:52.528+0000 7f7eca0fb000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:50:52.832 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:52.626+0000 7f7eca0fb000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:50:52.832 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:52.729+0000 7f7eca0fb000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: [21/Jun/2024:00:50:51] ENGINE Bus STARTING 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: [21/Jun/2024:00:50:51] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: [21/Jun/2024:00:50:51] ENGINE Error in HTTPServer.serve 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._run(expiration_interval) 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:50:53.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: s = self.context.wrap_socket( 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.sslsocket_class._create( 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.do_handshake() 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._sslobj.do_handshake() 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: [21/Jun/2024:00:50:52] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T00:50:53.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:52 smithi120 ceph-mon[25975]: [21/Jun/2024:00:50:52] ENGINE Bus STARTED 2024-06-21T00:50:53.225 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:53.002+0000 7f7eca0fb000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: [21/Jun/2024:00:50:51] ENGINE Bus STARTING 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: [21/Jun/2024:00:50:51] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: [21/Jun/2024:00:50:51] ENGINE Error in HTTPServer.serve 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-06-21T00:50:53.226 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: [21/Jun/2024:00:50:52] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T00:50:53.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[25136]: [21/Jun/2024:00:50:52] ENGINE Bus STARTED 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: [21/Jun/2024:00:50:51] ENGINE Bus STARTING 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: [21/Jun/2024:00:50:51] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: [21/Jun/2024:00:50:51] ENGINE Error in HTTPServer.serve 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:50:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: [21/Jun/2024:00:50:52] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T00:50:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:52 smithi027 ceph-mon[29814]: [21/Jun/2024:00:50:52] ENGINE Bus STARTED 2024-06-21T00:50:53.836 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:53.483+0000 7f7eca0fb000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:50:53.837 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=error ts=2024-06-21T00:50:53.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.27:8443//api/prometheus_receiver\": dial tcp 172.21.15.27:8443: connect: connection refused" 2024-06-21T00:50:53.837 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": dial tcp 172.21.15.27:8443: connect: connection refused" 2024-06-21T00:50:53.837 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.120:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.120 because it doesn't contain any IP SANs" 2024-06-21T00:50:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:53 smithi120 ceph-mon[25975]: mgrmap e24: x(active, since 3s) 2024-06-21T00:50:54.131 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-mon[25136]: mgrmap e24: x(active, since 3s) 2024-06-21T00:50:54.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:53 smithi027 ceph-mon[29814]: mgrmap e24: x(active, since 3s) 2024-06-21T00:50:54.132 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:54.029+0000 7f7eca0fb000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:50:54.132 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:54.132+0000 7f7eca0fb000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:50:54.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:54.447+0000 7f7eca0fb000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:50:54.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:54.549+0000 7f7eca0fb000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:50:55.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:54 smithi120 ceph-mon[25975]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:55.370 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-mon[25136]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:55.371 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-mon[29814]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:55.371 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:54 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:54.991+0000 7f7eca0fb000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:50:55.451 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:55 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:55.132+0000 7f7eca0fb000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:50:55.451 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:55 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:55.230+0000 7f7eca0fb000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:50:55.451 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:55 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:55.352+0000 7f7eca0fb000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:50:55.452 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:55 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:55.452+0000 7f7eca0fb000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:50:56.431 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:56.046+0000 7f7eca0fb000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:50:56.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:56.433+0000 7f7eca0fb000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:50:56] ENGINE Bus STARTING 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: CherryPy Checker: 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: The Application mounted at '' has an empty config. 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:50:56] ENGINE Serving on http://:::9283 2024-06-21T00:50:56.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: [21/Jun/2024:00:50:56] ENGINE Bus STARTED 2024-06-21T00:50:57.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:57.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: Standby manager daemon y started 2024-06-21T00:50:57.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.478 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=warn ts=2024-06-21T00:50:57.255Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.27:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.27 because it doesn't contain any IP SANs" 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: Standby manager daemon y started 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:50:57.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: Standby manager daemon y started 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:50:57.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/708882161' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:57.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:57 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.095 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:50:57 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:57] "GET /metrics HTTP/1.1" 200 34899 "" "Prometheus/2.33.4" 2024-06-21T00:50:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi027 to 2992M 2024-06-21T00:50:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-21T00:50:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:50:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Adjusting osd_memory_target on smithi120 to 3248M 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:50:58.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:50:58.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi027 to 2992M 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Adjusting osd_memory_target on smithi120 to 3248M 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi027 to 2992M 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: mgrmap e25: x(active, since 8s), standbys: y 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Adjusting osd_memory_target on smithi120 to 3248M 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:50:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:50:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:50:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:50:58.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:58 smithi027 systemd[1]: Stopping Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:50:58.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:58 smithi027 bash[58722]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter.a" found: no such container 2024-06-21T00:50:58.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:58 smithi027 podman[58734]: 2024-06-21 00:50:58.751789637 +0000 UTC m=+0.165146230 container died 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:50:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.27:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:50:59.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:50:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:50:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.27:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:50:59.393 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58734]: 2024-06-21 00:50:59.150780795 +0000 UTC m=+0.564137391 container cleanup 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.394 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 bash[58734]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-21T00:50:59.394 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 bash[58759]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter.a" found: no such container 2024-06-21T00:50:59.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[29814]: Reconfiguring daemon node-exporter.a on smithi027 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.396 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-21T00:50:59.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-mon[25136]: Reconfiguring daemon node-exporter.a on smithi027 2024-06-21T00:50:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:50:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='client.14805 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:50:59.619 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Reconfiguring node-exporter.a (dependencies changed)... 2024-06-21T00:50:59.620 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:50:59 smithi120 ceph-mon[25975]: Reconfiguring daemon node-exporter.a on smithi027 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58748]: 2024-06-21 00:50:59.392823951 +0000 UTC m=+0.641152121 container remove 7a46e91c0993bb3b070ea675baf37117f8b41c5a6f2c992c2e8814bb9e0c8dc2 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: Stopped Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 3.992s CPU time. 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: Starting Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:50:59.658 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58824]: 2024-06-21 00:50:59.532577066 +0000 UTC m=+0.017355808 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-21T00:50:59.659 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58824]: 2024-06-21 00:50:59.660510244 +0000 UTC m=+0.145289017 container create 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.969 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58824]: 2024-06-21 00:50:59.879554961 +0000 UTC m=+0.364333704 container init 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.969 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 podman[58824]: 2024-06-21 00:50:59.882562631 +0000 UTC m=+0.367341362 container start 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.885Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.885Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.886Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.886Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=arp 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-21T00:50:59.970 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-21T00:50:59.971 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=edac 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-21T00:50:59.972 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=os 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-21T00:50:59.973 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=stat 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=time 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=uname 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-21T00:50:59.974 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-21T00:50:59.975 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-21T00:50:59.975 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-21T00:50:59.975 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[58838]: ts=2024-06-21T00:50:59.887Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-21T00:50:59.975 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 bash[58824]: 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 2024-06-21T00:51:00.228 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 00:50:59 smithi027 systemd[1]: Started Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:01.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:01 smithi120 ceph-mon[25975]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:51:01.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:01 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:01 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:01 smithi120 ceph-mon[25975]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:51:01.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:01 smithi120 ceph-mon[25975]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:51:01.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[25136]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:51:01.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[25136]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[25136]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[29814]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[29814]: Reconfiguring alertmanager.a (dependencies changed)... 2024-06-21T00:51:01.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:01 smithi027 ceph-mon[29814]: Reconfiguring daemon alertmanager.a on smithi027 2024-06-21T00:51:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:03 smithi120 ceph-mon[25975]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:51:03.158 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:03 smithi027 ceph-mon[25136]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:51:03.158 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:03 smithi027 ceph-mon[29814]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:51:03.159 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 systemd[1]: Stopping Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:03.159 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 bash[59008]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-21T00:51:03.459 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[49561]: level=info ts=2024-06-21T00:51:03.160Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-21T00:51:03.459 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 podman[59018]: 2024-06-21 00:51:03.191174715 +0000 UTC m=+0.135276692 container died aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:03.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 podman[59018]: 2024-06-21 00:51:03.551410802 +0000 UTC m=+0.495512773 container cleanup aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:03.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 bash[59018]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-21T00:51:03.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 bash[59045]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager.a" found: no such container 2024-06-21T00:51:03.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:03.506Z caller=notifier.go:526 level=error component=notifier alertmanager=http://172.21.15.27:9093/api/v2/alerts count=3 msg="Error sending alert" err="Post \"http://172.21.15.27:9093/api/v2/alerts\": dial tcp 172.21.15.27:9093: connect: connection refused" 2024-06-21T00:51:04.138 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 podman[59031]: 2024-06-21 00:51:03.827746461 +0000 UTC m=+0.633081990 container remove aa8c1dea32b85d6703f862bf7e6ab6af14a20f473f3d31903af50e173bb89f24 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 podman[59031]: 2024-06-21 00:51:03.895224578 +0000 UTC m=+0.700560107 volume remove ad25015aec00368d0ebac31712b53d7a5a38654bf720c79a54dd580842c6cda9 2024-06-21T00:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-21T00:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 systemd[1]: Stopped Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.086s CPU time. 2024-06-21T00:51:04.139 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:03 smithi027 systemd[1]: Starting Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:04.460 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 podman[59108]: 2024-06-21 00:51:04.04381073 +0000 UTC m=+0.018337587 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-06-21T00:51:04.460 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 podman[59108]: 2024-06-21 00:51:04.171080769 +0000 UTC m=+0.145607614 volume create 1788fbfcf5947efe53416d5f1a0bf3b71854904209cd7894439a3acc26f89cc5 2024-06-21T00:51:04.461 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 podman[59108]: 2024-06-21 00:51:04.263175405 +0000 UTC m=+0.237702293 container create 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:04.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 podman[59108]: 2024-06-21 00:51:04.574583169 +0000 UTC m=+0.549110027 container init 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 podman[59108]: 2024-06-21 00:51:04.57782047 +0000 UTC m=+0.552347317 container start 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.589Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.589Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.591Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.592Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.619Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.619Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.621Z caller=main.go:518 msg=Listening address=:9093 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:04.621Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 bash[59108]: 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 2024-06-21T00:51:04.980 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:04 smithi027 systemd[1]: Started Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:05.291 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:06.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:06.288 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:06.288 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:06.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:06.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:06.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:06 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:06 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:06.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:06 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:06.882 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:06.592Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000336523s 2024-06-21T00:51:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:07 smithi027 ceph-mon[25136]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:07 smithi027 ceph-mon[29814]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:07.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-mon[25975]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:07] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-06-21T00:51:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:51:07.910+0000 7f39c9fce640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:51:08.117 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:08.118 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:08.119 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.wait_async( 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return future.result(timeout) 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.__get_result() 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise self._exception 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:08.120 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: out, err, code = await self._run_cephadm( 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise OrchestratorError( 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:08.121 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:08.122 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:51:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[25365]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:51:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:09 smithi120 ceph-mon[25975]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:09 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.117 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.118 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:09.119 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.120 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.121 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:09 smithi120 ceph-mon[25975]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:51:09.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:09 smithi120 ceph-mon[25975]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[25136]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:09.480 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:51:09.481 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.482 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[25136]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[25136]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[29814]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:51:09.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.484 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:09.485 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.486 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:09.487 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:09.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[29814]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:51:09.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:09 smithi027 ceph-mon[29814]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:51:11.278 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-mon[25975]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:11.278 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:11.278 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 bash[48980]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:51:11.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:11 smithi027 ceph-mon[25136]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:11.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:11 smithi027 ceph-mon[29814]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:11.570 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T00:51:11.570 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T00:51:11.570 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T00:51:11.570 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T00:51:11.570 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.280Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.281Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.281Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[42757]: ts=2024-06-21T00:51:11.281Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T00:51:11.571 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 podman[48991]: 2024-06-21 00:51:11.290256158 +0000 UTC m=+0.125342213 container died d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:11.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 podman[48991]: 2024-06-21 00:51:11.663027492 +0000 UTC m=+0.498113548 container cleanup d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:11.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 bash[48991]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T00:51:11.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 bash[49018]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:51:12.276 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 podman[49004]: 2024-06-21 00:51:11.914392149 +0000 UTC m=+0.625108277 container remove d65c4d07f53c2653bef36e58ba7c5b25eeffa1e06bec3cc6e159b6a6ee1dba73 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:12.276 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T00:51:12.276 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:12.276 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.959s CPU time. 2024-06-21T00:51:12.277 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:11 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:12.277 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 podman[49079]: 2024-06-21 00:51:12.054963163 +0000 UTC m=+0.017946011 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-21T00:51:12.277 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 podman[49079]: 2024-06-21 00:51:12.157242408 +0000 UTC m=+0.120225254 container create b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:12.607 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 podman[49079]: 2024-06-21 00:51:12.401359581 +0000 UTC m=+0.364342428 container init b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:12.607 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 podman[49079]: 2024-06-21 00:51:12.404860792 +0000 UTC m=+0.367843640 container start b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:51:12.607 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T00:51:12.607 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.427Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.429Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.429Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.431Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.432Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.432Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.51µs 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.432Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 bash[49079]: b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 2024-06-21T00:51:12.608 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:12.609 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.609Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-06-21T00:51:12.866 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:12 smithi120 systemd[1]: Stopping Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:12.866 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:12 smithi120 bash[49202]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter.b" found: no such container 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.847Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.847Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.847Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.982µs wal_replay_duration=414.714205ms total_replay_duration=414.759311ms 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.851Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.851Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T00:51:12.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.851Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T00:51:13.154 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:12 smithi120 podman[49213]: 2024-06-21 00:51:12.981932037 +0000 UTC m=+0.147745126 container died 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:13.155 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.875Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.711678ms db_storage=964ns remote_storage=1.61µs web_handler=603ns query_engine=768ns scrape=556.583µs scrape_sd=172.419µs notify=17.948µs notify_sd=19.359µs rules=22.408331ms 2024-06-21T00:51:13.155 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:12.875Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T00:51:13.512 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49213]: 2024-06-21 00:51:13.304669105 +0000 UTC m=+0.470482184 container cleanup 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:13.512 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 bash[49213]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-21T00:51:13.780 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-mon[25975]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:13.780 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.780 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.780 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-mon[25975]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-21T00:51:13.780 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-mon[25975]: Reconfiguring daemon node-exporter.b on smithi120 2024-06-21T00:51:13.780 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49226]: 2024-06-21 00:51:13.514493915 +0000 UTC m=+0.536574416 container remove 7c4e32f60c7413982f9c8cf2987cd521bbfa0b58cfdc0b55696bd391ec8d233d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:13.780 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-21T00:51:13.780 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 bash[49239]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter.b" found: no such container 2024-06-21T00:51:13.781 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-21T00:51:13.781 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 systemd[1]: Stopped Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:13.781 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 3.815s CPU time. 2024-06-21T00:51:13.781 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 systemd[1]: Starting Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[25136]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[25136]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[25136]: Reconfiguring daemon node-exporter.b on smithi120 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[29814]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[29814]: Reconfiguring node-exporter.b (dependencies changed)... 2024-06-21T00:51:13.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:13 smithi027 ceph-mon[29814]: Reconfiguring daemon node-exporter.b on smithi120 2024-06-21T00:51:14.057 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49307]: 2024-06-21 00:51:13.685833926 +0000 UTC m=+0.019228329 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-06-21T00:51:14.057 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49307]: 2024-06-21 00:51:13.790523222 +0000 UTC m=+0.123917608 container create 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49307]: 2024-06-21 00:51:13.98451765 +0000 UTC m=+0.317912042 container init 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 podman[49307]: 2024-06-21 00:51:13.987345344 +0000 UTC m=+0.320739724 container start 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.990Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.990Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=arp 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=bcache 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=bonding 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=btrfs 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=conntrack 2024-06-21T00:51:14.058 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=cpu 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=diskstats 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=dmi 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=edac 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=entropy 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=filefd 2024-06-21T00:51:14.059 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=filesystem 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=hwmon 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=infiniband 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=ipvs 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=loadavg 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=mdadm 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=meminfo 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=netclass 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=netdev 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=netstat 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=nfs 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=nfsd 2024-06-21T00:51:14.060 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=nvme 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=os 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=pressure 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=rapl 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=schedstat 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=sockstat 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=softnet 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=stat 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=tapestats 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=textfile 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=time 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=uname 2024-06-21T00:51:14.061 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=vmstat 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=xfs 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:115 level=info collector=zfs 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.991Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[49323]: ts=2024-06-21T00:51:13.992Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:14 smithi120 bash[49307]: 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff 2024-06-21T00:51:14.062 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 00:51:14 smithi120 systemd[1]: Started Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:14.981 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 00:51:14 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T00:51:14.595Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002778634s 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: Reconfiguring grafana.a (dependencies changed)... 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:15 smithi120 ceph-mon[25975]: Reconfiguring daemon grafana.a on smithi120 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: Reconfiguring grafana.a (dependencies changed)... 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[25136]: Reconfiguring daemon grafana.a on smithi120 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: Reconfiguring grafana.a (dependencies changed)... 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-21T00:51:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:15 smithi027 ceph-mon[29814]: Reconfiguring daemon grafana.a on smithi120 2024-06-21T00:51:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:16 smithi027 ceph-mon[25136]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:16 smithi027 ceph-mon[29814]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:16 smithi120 ceph-mon[25975]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:17.117 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:16 smithi120 systemd[1]: Stopping Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:17.117 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:16 smithi120 bash[49493]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana.a" found: no such container 2024-06-21T00:51:17.117 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:51:16+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-06-21T00:51:17.117 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[41983]: t=2024-06-21T00:51:16+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2024-06-21T00:51:17.117 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:16 smithi120 podman[49503]: 2024-06-21 00:51:16.873812997 +0000 UTC m=+0.150141026 container died da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Paul Cuzner , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, name=ubi8, release=236.1648460182, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-06-21T00:51:17.419 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49503]: 2024-06-21 00:51:17.203339118 +0000 UTC m=+0.479667140 container cleanup da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, distribution-scope=public, architecture=x86_64, io.buildah.version=1.24.2, release=236.1648460182, version=8.5, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=) 2024-06-21T00:51:17.419 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 bash[49503]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a 2024-06-21T00:51:17.701 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49515]: 2024-06-21 00:51:17.421394666 +0000 UTC m=+0.544949589 container remove da2526921115f0154bde47ff662fc7e7fa0bd75a46ea51bc2b9fd97cedfcea37 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , release=236.1648460182, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, distribution-scope=public, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc.) 2024-06-21T00:51:17.701 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 bash[49531]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana.a" found: no such container 2024-06-21T00:51:17.701 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-21T00:51:17.702 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 systemd[1]: Stopped Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:17.702 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 3.282s CPU time. 2024-06-21T00:51:17.702 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 systemd[1]: Starting Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:51:17.981 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49592]: 2024-06-21 00:51:17.605388126 +0000 UTC m=+0.017078834 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-06-21T00:51:17.981 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49592]: 2024-06-21 00:51:17.725372564 +0000 UTC m=+0.137063261 container create 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=236.1648460182, maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-06-21T00:51:18.306 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STOPPING 2024-06-21T00:51:18.306 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T00:51:18.306 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STOPPED 2024-06-21T00:51:18.306 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49592]: 2024-06-21 00:51:17.983637545 +0000 UTC m=+0.395328250 container init 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, io.openshift.tags=base rhel8, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, distribution-scope=public, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.buildah.version=1.24.2, release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, architecture=x86_64) 2024-06-21T00:51:18.306 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:17 smithi120 podman[49592]: 2024-06-21 00:51:17.986445101 +0000 UTC m=+0.398135799 container start 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, vendor=Red Hat, Inc., version=8.5, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=) 2024-06-21T00:51:18.306 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-06-21T00:51:18.306 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-06-21T00:51:18.307 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="App mode production" logger=settings 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.073667ms 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 bash[49592]: 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 systemd[1]: Started Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:51:18.308 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-06-21T00:51:18.309 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T00:51:18+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-06-21T00:51:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STARTING 2024-06-21T00:51:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:18] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-06-21T00:51:18.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Serving on http://:::9283 2024-06-21T00:51:18.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STARTED 2024-06-21T00:51:18.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STOPPING 2024-06-21T00:51:18.913 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:19.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STOPPED 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:18] ENGINE Bus STARTING 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Serving on http://:::9283 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Bus STARTED 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Bus STOPPING 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.374 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:51:19.375 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi120.front.sepia.ceph.com:9095"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T00:51:19.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi120.front.sepia.ceph.com:3000"}]: dispatch 2024-06-21T00:51:19.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:19.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:51:19.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T00:51:19.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Bus STOPPED 2024-06-21T00:51:19.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Bus STARTING 2024-06-21T00:51:19.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Serving on http://:::9283 2024-06-21T00:51:19.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:51:19] ENGINE Bus STARTED 2024-06-21T00:51:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/319422156' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:20 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]: dispatch 2024-06-21T00:51:20.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/319422156' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:20.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:20 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]: dispatch 2024-06-21T00:51:20.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/319422156' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:20.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:20 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]: dispatch 2024-06-21T00:51:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:51:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]': finished 2024-06-21T00:51:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: osdmap e97: 8 total, 8 up, 8 in 2024-06-21T00:51:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1062459122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:21 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:21.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:51:21.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]': finished 2024-06-21T00:51:21.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: osdmap e97: 8 total, 8 up, 8 in 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1062459122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/235311297' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3971766118"}]': finished 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: osdmap e97: 8 total, 8 up, 8 in 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1062459122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:21.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:21 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]: dispatch 2024-06-21T00:51:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]': finished 2024-06-21T00:51:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[25136]: osdmap e98: 8 total, 8 up, 8 in 2024-06-21T00:51:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/258964950' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:22.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]': finished 2024-06-21T00:51:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[29814]: osdmap e98: 8 total, 8 up, 8 in 2024-06-21T00:51:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/258964950' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:22 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:22.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:22 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/613211723"}]': finished 2024-06-21T00:51:22.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:22 smithi120 ceph-mon[25975]: osdmap e98: 8 total, 8 up, 8 in 2024-06-21T00:51:22.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:22 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/258964950' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:22.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:22 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]: dispatch 2024-06-21T00:51:23.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:51:23.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]': finished 2024-06-21T00:51:23.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: osdmap e99: 8 total, 8 up, 8 in 2024-06-21T00:51:23.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3120655805' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]': finished 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: osdmap e99: 8 total, 8 up, 8 in 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3120655805' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:23 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/3654643913"}]': finished 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: osdmap e99: 8 total, 8 up, 8 in 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.504 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3120655805' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.505 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]: dispatch 2024-06-21T00:51:23.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:24.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]': finished 2024-06-21T00:51:24.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[29814]: osdmap e100: 8 total, 8 up, 8 in 2024-06-21T00:51:24.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]: dispatch 2024-06-21T00:51:24.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]': finished 2024-06-21T00:51:24.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[25136]: osdmap e100: 8 total, 8 up, 8 in 2024-06-21T00:51:24.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:24 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]: dispatch 2024-06-21T00:51:24.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:24 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2573480600"}]': finished 2024-06-21T00:51:24.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:24 smithi120 ceph-mon[25975]: osdmap e100: 8 total, 8 up, 8 in 2024-06-21T00:51:24.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:24 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]: dispatch 2024-06-21T00:51:25.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-mon[25975]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:51:25.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]': finished 2024-06-21T00:51:25.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-mon[25975]: osdmap e101: 8 total, 8 up, 8 in 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:51:25.262+0000 7f39c9fce640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:25.618 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:25.619 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.wait_async( 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return future.result(timeout) 2024-06-21T00:51:25.620 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.__get_result() 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise self._exception 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: out, err, code = await self._run_cephadm( 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise OrchestratorError( 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:51:25.621 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:25.622 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:25.623 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:25.623 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:25.623 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:25.623 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:25.623 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:25.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[29814]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:51:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]': finished 2024-06-21T00:51:25.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[29814]: osdmap e101: 8 total, 8 up, 8 in 2024-06-21T00:51:25.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[25136]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:51:25.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3572626122' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1067287154"}]': finished 2024-06-21T00:51:25.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:25 smithi027 ceph-mon[25136]: osdmap e101: 8 total, 8 up, 8 in 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:26 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.729 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.730 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:26.731 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.732 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:26 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.734 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:26.735 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.736 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.737 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.738 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:26 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T00:51:26.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:51:26.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:51:26.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:51:26.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:51:26.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:51:26.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:51:26.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:51:27.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:27 smithi027 ceph-mon[25136]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:27.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:27 smithi027 ceph-mon[29814]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:27.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:27 smithi120 ceph-mon[25975]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:28.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:28] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-06-21T00:51:29.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:29 smithi027 ceph-mon[25136]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:51:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:29 smithi027 ceph-mon[29814]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:51:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:29 smithi120 ceph-mon[25975]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:51:31.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:31 smithi027 ceph-mon[25136]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-06-21T00:51:31.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:31 smithi027 ceph-mon[29814]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-06-21T00:51:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:31 smithi120 ceph-mon[25975]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 531 B/s rd, 0 op/s 2024-06-21T00:51:33.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:33 smithi027 ceph-mon[25136]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 485 B/s rd, 0 op/s 2024-06-21T00:51:33.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:33 smithi027 ceph-mon[29814]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 485 B/s rd, 0 op/s 2024-06-21T00:51:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:33 smithi120 ceph-mon[25975]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 485 B/s rd, 0 op/s 2024-06-21T00:51:33.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:33.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:35.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:35 smithi027 ceph-mon[25136]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-06-21T00:51:35.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:35.504 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:35 smithi027 ceph-mon[29814]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-06-21T00:51:35.504 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:35 smithi120 ceph-mon[25975]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 409 B/s rd, 0 op/s 2024-06-21T00:51:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:36.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:36 smithi027 ceph-mon[25136]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 358 B/s rd, 0 op/s 2024-06-21T00:51:36.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:36 smithi027 ceph-mon[29814]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 358 B/s rd, 0 op/s 2024-06-21T00:51:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:36 smithi120 ceph-mon[25975]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 358 B/s rd, 0 op/s 2024-06-21T00:51:37.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:37 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/483059541' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:37.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:37 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]: dispatch 2024-06-21T00:51:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:37 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/483059541' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:37.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:37 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]: dispatch 2024-06-21T00:51:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:37 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/483059541' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:51:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:37 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]: dispatch 2024-06-21T00:51:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:38] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-06-21T00:51:38.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]': finished 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[25136]: osdmap e102: 8 total, 8 up, 8 in 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[25136]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]: dispatch 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]': finished 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[29814]: osdmap e102: 8 total, 8 up, 8 in 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[29814]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:38.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:38 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]: dispatch 2024-06-21T00:51:39.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2525520933' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1067287154"}]': finished 2024-06-21T00:51:39.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-mon[25975]: osdmap e102: 8 total, 8 up, 8 in 2024-06-21T00:51:39.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-mon[25975]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:39.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]: dispatch 2024-06-21T00:51:39.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:39 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]': finished 2024-06-21T00:51:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:39 smithi027 ceph-mon[25136]: osdmap e103: 8 total, 8 up, 8 in 2024-06-21T00:51:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:39 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]': finished 2024-06-21T00:51:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:39 smithi027 ceph-mon[29814]: osdmap e103: 8 total, 8 up, 8 in 2024-06-21T00:51:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:39 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/306019486' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2823981577"}]': finished 2024-06-21T00:51:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:39 smithi120 ceph-mon[25975]: osdmap e103: 8 total, 8 up, 8 in 2024-06-21T00:51:40.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:40 smithi027 ceph-mon[25136]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:40.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:40 smithi027 ceph-mon[29814]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:41.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:40 smithi120 ceph-mon[25975]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:51:43.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:42 smithi027 ceph-mon[25136]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:43.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:42 smithi027 ceph-mon[29814]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:43.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:42 smithi120 ceph-mon[25975]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:43.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:43.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:44 smithi027 ceph-mon[25136]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:44 smithi027 ceph-mon[29814]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:45.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:44 smithi120 ceph-mon[25975]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T00:51:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:46 smithi027 ceph-mon[25136]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-06-21T00:51:46.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:46 smithi027 ceph-mon[25136]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:46 smithi027 ceph-mon[29814]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-06-21T00:51:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:46 smithi027 ceph-mon[29814]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:46 smithi120 ceph-mon[25975]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-06-21T00:51:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:46 smithi120 ceph-mon[25975]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:48] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-06-21T00:51:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:48 smithi120 ceph-mon[25975]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:49.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:49.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:48 smithi027 ceph-mon[25136]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:49.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:48 smithi027 ceph-mon[29814]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:51:51.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:50 smithi027 ceph-mon[29814]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-06-21T00:51:51.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:51.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:50 smithi027 ceph-mon[25136]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-06-21T00:51:51.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:50 smithi120 ceph-mon[25975]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-06-21T00:51:51.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:51:53.169 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:52 smithi120 ceph-mon[25975]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:52 smithi027 ceph-mon[25136]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:52 smithi027 ceph-mon[29814]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:53.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:53.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:55.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:54 smithi027 ceph-mon[25136]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:54 smithi027 ceph-mon[29814]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:55.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:54 smithi120 ceph-mon[25975]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:51:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:56 smithi027 ceph-mon[29814]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:56.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:56 smithi027 ceph-mon[29814]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:56.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:56 smithi027 ceph-mon[25136]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:56.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:56 smithi027 ceph-mon[25136]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:56 smithi120 ceph-mon[25975]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:56 smithi120 ceph-mon[25975]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:51:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:51:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:51:58] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-06-21T00:51:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:51:58 smithi120 ceph-mon[25975]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:59.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:51:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:51:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:51:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:51:58 smithi027 ceph-mon[25136]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:51:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:51:58 smithi027 ceph-mon[29814]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:01.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:00 smithi027 ceph-mon[25136]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:01.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:00 smithi027 ceph-mon[29814]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:01.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:00 smithi120 ceph-mon[25975]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:02 smithi027 ceph-mon[25136]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:02 smithi027 ceph-mon[29814]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:02 smithi120 ceph-mon[25975]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:03.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:04 smithi027 ceph-mon[29814]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:04 smithi027 ceph-mon[25136]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:04 smithi120 ceph-mon[25975]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:06.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:06.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:06 smithi027 ceph-mon[25136]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:07.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:06 smithi027 ceph-mon[25136]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:06 smithi027 ceph-mon[29814]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:06 smithi027 ceph-mon[29814]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:07.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:06 smithi120 ceph-mon[25975]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:07.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:06 smithi120 ceph-mon[25975]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:08] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-06-21T00:52:08.914 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:09.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:08 smithi027 ceph-mon[25136]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:09.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:08 smithi027 ceph-mon[29814]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:09.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:08 smithi120 ceph-mon[25975]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:10 smithi027 ceph-mon[25136]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:11.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:10 smithi027 ceph-mon[29814]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:11.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:10 smithi120 ceph-mon[25975]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:12 smithi027 ceph-mon[25136]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:12 smithi027 ceph-mon[29814]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:13.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:12 smithi120 ceph-mon[25975]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:13.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:13.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:14 smithi027 ceph-mon[25136]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:14 smithi027 ceph-mon[29814]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:14 smithi120 ceph-mon[25975]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:16 smithi027 ceph-mon[25136]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:16 smithi027 ceph-mon[25136]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:16 smithi027 ceph-mon[29814]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:16 smithi027 ceph-mon[29814]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:16 smithi120 ceph-mon[25975]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:16 smithi120 ceph-mon[25975]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:18.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:18] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-06-21T00:52:19.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:18 smithi120 ceph-mon[25975]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:19.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:19.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:18 smithi027 ceph-mon[25136]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:19.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:18 smithi027 ceph-mon[29814]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:20 smithi027 ceph-mon[25136]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:20 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:20 smithi027 ceph-mon[29814]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:20 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:20 smithi120 ceph-mon[25975]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:21.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:20 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:23.169 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:22 smithi120 ceph-mon[25975]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:22 smithi027 ceph-mon[25136]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:22 smithi027 ceph-mon[29814]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:23.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:25.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:24 smithi027 ceph-mon[25136]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:24 smithi027 ceph-mon[29814]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:24 smithi120 ceph-mon[25975]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:25 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:52:26.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:25 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:52:26.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:25 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:52:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:26 smithi027 ceph-mon[25136]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:26 smithi027 ceph-mon[25136]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:26 smithi027 ceph-mon[29814]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:26 smithi027 ceph-mon[29814]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:27.250 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:26 smithi120 ceph-mon[25975]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:27.250 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:26 smithi120 ceph-mon[25975]: from='client.25039 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:28] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-06-21T00:52:28.909 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:28 smithi027 ceph-mon[25136]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:28 smithi027 ceph-mon[29814]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:29.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:28 smithi120 ceph-mon[25975]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:30 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:52:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:52:30.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:30 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:52:31.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:31 smithi027 ceph-mon[25136]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:31.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:31 smithi027 ceph-mon[29814]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:31 smithi120 ceph-mon[25975]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:52:32.505+0000 7f39c9fce640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:32.867 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:32.868 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.wait_async( 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:52:32.869 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return future.result(timeout) 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.__get_result() 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise self._exception 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: out, err, code = await self._run_cephadm( 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise OrchestratorError( 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:52:32.870 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:32.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:32.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:32.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:32.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:32.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:32.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:33 smithi120 ceph-mon[25975]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:33 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:52:33.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.872 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:33 smithi027 ceph-mon[25136]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:33 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.980 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T00:52:33.981 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.982 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.983 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:33 smithi027 ceph-mon[29814]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:33 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.984 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:52:33.985 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:52:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:52:33.987 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:52:33.988 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:52:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:35 smithi120 ceph-mon[25975]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:35 smithi027 ceph-mon[25136]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:35 smithi027 ceph-mon[29814]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:36.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:36 smithi027 ceph-mon[25136]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-06-21T00:52:36.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:36 smithi027 ceph-mon[29814]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-06-21T00:52:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:36 smithi120 ceph-mon[25975]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-06-21T00:52:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:38] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-06-21T00:52:38.908 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:38 smithi027 ceph-mon[25136]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:39.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:38 smithi027 ceph-mon[29814]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:39.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:38 smithi120 ceph-mon[25975]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:41.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:40 smithi027 ceph-mon[25136]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:41.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:40 smithi027 ceph-mon[29814]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:40 smithi120 ceph-mon[25975]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:43.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:42 smithi027 ceph-mon[25136]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:43.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:42 smithi027 ceph-mon[29814]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:43.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:42 smithi120 ceph-mon[25975]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-06-21T00:52:43.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:44 smithi027 ceph-mon[25136]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-21T00:52:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:44 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/603879983' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:52:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:44 smithi027 ceph-mon[29814]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-21T00:52:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:44 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/603879983' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:52:45.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:44 smithi120 ceph-mon[25975]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-06-21T00:52:45.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:44 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/603879983' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:52:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:46 smithi027 ceph-mon[25136]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:52:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:46 smithi027 ceph-mon[29814]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:52:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:46 smithi120 ceph-mon[25975]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T00:52:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:48] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-06-21T00:52:48.911 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:49.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:48 smithi027 ceph-mon[25136]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:49.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:48 smithi027 ceph-mon[29814]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:49.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:48 smithi120 ceph-mon[25975]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:51.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:50 smithi027 ceph-mon[25136]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:51.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:51.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:50 smithi027 ceph-mon[29814]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:51.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:50 smithi120 ceph-mon[25975]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-21T00:52:51.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:52:53.169 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:52 smithi120 ceph-mon[25975]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:52 smithi027 ceph-mon[25136]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:52 smithi027 ceph-mon[29814]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:53.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:53.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:54 smithi027 ceph-mon[25136]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:54 smithi027 ceph-mon[25136]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:54 smithi027 ceph-mon[29814]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:54 smithi027 ceph-mon[29814]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:55.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:54 smithi120 ceph-mon[25975]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-06-21T00:52:55.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:54 smithi120 ceph-mon[25975]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:52:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:56 smithi027 ceph-mon[25136]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:52:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:56 smithi027 ceph-mon[29814]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:52:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:56 smithi120 ceph-mon[25975]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T00:52:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:52:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:52:58] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-06-21T00:52:58.914 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:52:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:52:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:52:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:52:58 smithi027 ceph-mon[25136]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:59.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:52:58 smithi027 ceph-mon[29814]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:52:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:52:58 smithi120 ceph-mon[25975]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:00 smithi027 ceph-mon[25136]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:00 smithi027 ceph-mon[29814]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:01.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:00 smithi120 ceph-mon[25975]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:02 smithi027 ceph-mon[25136]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:02 smithi027 ceph-mon[29814]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:03.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:02 smithi120 ceph-mon[25975]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:03.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:04 smithi027 ceph-mon[25136]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:04 smithi027 ceph-mon[25136]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:05.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:04 smithi027 ceph-mon[29814]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:05.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:04 smithi027 ceph-mon[29814]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:04 smithi120 ceph-mon[25975]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:05.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:04 smithi120 ceph-mon[25975]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:06.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:06.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:06 smithi027 ceph-mon[25136]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:06 smithi027 ceph-mon[29814]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:07.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:06 smithi120 ceph-mon[25975]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:08] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-06-21T00:53:08.928 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:09.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:08 smithi027 ceph-mon[25136]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:09.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:08 smithi027 ceph-mon[29814]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:09.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:08 smithi120 ceph-mon[25975]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:10 smithi027 ceph-mon[25136]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:11.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:10 smithi027 ceph-mon[29814]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:11.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:10 smithi120 ceph-mon[25975]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:12 smithi027 ceph-mon[29814]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:12 smithi027 ceph-mon[25136]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:13.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:12 smithi120 ceph-mon[25975]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:13.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:15.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:14 smithi027 ceph-mon[29814]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:15.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:14 smithi027 ceph-mon[29814]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:15.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:14 smithi027 ceph-mon[25136]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:15.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:14 smithi027 ceph-mon[25136]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:14 smithi120 ceph-mon[25975]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:14 smithi120 ceph-mon[25975]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:16 smithi027 ceph-mon[25136]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:16.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:16 smithi027 ceph-mon[29814]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:16 smithi120 ceph-mon[25975]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:18] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-06-21T00:53:18.920 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:19.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:18 smithi027 ceph-mon[25136]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:19.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:18 smithi027 ceph-mon[29814]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:19.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:18 smithi120 ceph-mon[25975]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:21.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:20 smithi027 ceph-mon[29814]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:20 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:20 smithi027 ceph-mon[25136]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:20 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:20 smithi120 ceph-mon[25975]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:20 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:22 smithi027 ceph-mon[25136]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:22 smithi027 ceph-mon[29814]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:22 smithi120 ceph-mon[25975]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:23.868 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:24 smithi027 ceph-mon[29814]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:24 smithi027 ceph-mon[29814]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:25.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:24 smithi027 ceph-mon[25136]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:25.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:24 smithi027 ceph-mon[25136]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:24 smithi120 ceph-mon[25975]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:24 smithi120 ceph-mon[25975]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:26.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:26 smithi027 ceph-mon[25136]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:26.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:26 smithi027 ceph-mon[29814]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:26 smithi120 ceph-mon[25975]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:28] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-06-21T00:53:28.923 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:28 smithi027 ceph-mon[25136]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:28 smithi027 ceph-mon[29814]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:29.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:28 smithi120 ceph-mon[25975]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:30 smithi027 ceph-mon[25136]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:30 smithi027 ceph-mon[29814]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:31.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:30 smithi120 ceph-mon[25975]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:33.219 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:32 smithi120 ceph-mon[25975]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:33.239 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:32 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:33.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:32 smithi027 ceph-mon[25136]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:33.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:32 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:33.243 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:32 smithi027 ceph-mon[29814]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:33.243 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:32 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:33.854 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:35.199 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:34 smithi120 ceph-mon[25975]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:35.199 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:34 smithi120 ceph-mon[25975]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:34 smithi027 ceph-mon[29814]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:34 smithi027 ceph-mon[29814]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:34 smithi027 ceph-mon[25136]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:34 smithi027 ceph-mon[25136]: from='client.25078 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:53:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:37 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:53:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:37 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:38.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:38 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:53:38.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:38 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:53:38.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:38 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:53:38.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:38 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:53:38.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:38 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:53:38.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:38 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:53:38.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:38] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:53:39.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:39.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:39 smithi027 ceph-mon[25136]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:39.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:39 smithi027 ceph-mon[29814]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-mon[25975]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:53:39.984+0000 7f39c9fce640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:53:40.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:40.368 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.wait_async( 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:53:40.369 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return future.result(timeout) 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.__get_result() 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise self._exception 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: out, err, code = await self._run_cephadm( 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise OrchestratorError( 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:40.370 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:40.371 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:40.372 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:41 smithi120 ceph-mon[25975]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:41 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.618 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:53:41.619 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:53:41.620 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.621 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:41 smithi027 ceph-mon[25136]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:41 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.729 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.730 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:53:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:53:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:41 smithi027 ceph-mon[29814]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:41 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.733 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.734 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:53:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:53:41.736 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:53:41.737 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:53:41.738 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:53:43.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:43 smithi120 ceph-mon[25975]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:43.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:43.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:43 smithi027 ceph-mon[25136]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:43.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:43 smithi027 ceph-mon[29814]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:53:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:45 smithi120 ceph-mon[25975]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:45 smithi027 ceph-mon[25136]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:45.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:45 smithi027 ceph-mon[29814]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:46 smithi027 ceph-mon[25136]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:46 smithi027 ceph-mon[29814]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:46 smithi120 ceph-mon[25975]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:53:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:48] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:53:48.930 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:49.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:48 smithi027 ceph-mon[25136]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:49.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:48 smithi027 ceph-mon[29814]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:49.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:48 smithi120 ceph-mon[25975]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:51.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:50 smithi027 ceph-mon[25136]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:51.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:51.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:50 smithi027 ceph-mon[29814]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:51.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:50 smithi120 ceph-mon[25975]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:53:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:51 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2831522503' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:53:51.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:51 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2831522503' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:53:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:51 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2831522503' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:53:52.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:52 smithi027 ceph-mon[25136]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:52.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:52 smithi027 ceph-mon[29814]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:53.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-21T00:53:53.273 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:52 smithi120 ceph-mon[25975]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T00:53:53.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:54.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:54 smithi027 ceph-mon[25136]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:53:54.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:54 smithi027 ceph-mon[29814]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:53:55.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:54 smithi120 ceph-mon[25975]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:53:55.465 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled to redeploy mgr.y on host 'smithi027' 2024-06-21T00:53:56.064 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1465349344' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.064 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:53:56.064 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: Schedule redeploy daemon mgr.y 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1465349344' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: Schedule redeploy daemon mgr.y 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:55 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps --refresh' 2024-06-21T00:53:56.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1465349344' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:53:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: Schedule redeploy daemon mgr.y 2024-06-21T00:53:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:56.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:55 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:53:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:56 smithi027 ceph-mon[25136]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:56.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:56 smithi027 ceph-mon[29814]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:57.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:56 smithi120 ceph-mon[25975]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:58.617 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:53:58] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:53:58.952 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:53:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:53:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[25136]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[29814]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:58 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:59.365 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 90s ago 10m 14.1M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (2m) 23s ago 10m 47.6M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 90s ago 9m 48.2M - 3.5 e1d6a67b021e c5aff5904914 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (5m) 23s ago 15m 502M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283 running (17m) 90s ago 17m 410M - 17.2.0 e1d6a67b021e 53e9dc04bcf4 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (17m) 90s ago 17m 55.4M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (15m) 23s ago 15m 47.3M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (15m) 90s ago 15m 47.2M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 90s ago 11m 16.7M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (2m) 23s ago 11m 17.0M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (15m) 90s ago 15m 52.3M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T00:53:59.366 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (14m) 90s ago 14m 55.4M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (14m) 90s ago 14m 47.6M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (13m) 90s ago 13m 51.8M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (13m) 23s ago 13m 52.4M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (13m) 23s ago 13m 51.7M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (12m) 23s ago 12m 48.7M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (12m) 23s ago 12m 51.5M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 23s ago 11m 71.1M - 2.33.4 514e6a882f6e b952bfcfeb28 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (10m) 90s ago 10m 75.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T00:53:59.367 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (10m) 23s ago 10m 73.4M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T00:53:59.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-mon[25975]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:53:59.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:53:59.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:53:59.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:53:59.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:58 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: Deploying daemon mgr.y on smithi027 2024-06-21T00:54:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[25136]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: Deploying daemon mgr.y on smithi027 2024-06-21T00:54:00.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:53:59 smithi027 ceph-mon[29814]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:54:00.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T00:54:00.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T00:54:00.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:00.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: Deploying daemon mgr.y on smithi027 2024-06-21T00:54:00.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:53:59 smithi120 ceph-mon[25975]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:54:00.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T00:54:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:00 smithi027 ceph-mon[25136]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:54:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:00 smithi027 ceph-mon[29814]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:54:01.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:00 smithi120 ceph-mon[25975]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T00:54:01.576 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:01 smithi027 systemd[1]: Stopping Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:54:02.199 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:01 smithi027 ceph-mon[25136]: from='client.15195 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:02.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:01 smithi027 ceph-mon[29814]: from='client.15195 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:02.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:01 smithi120 ceph-mon[25975]: from='client.15195 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:03.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:02 smithi027 ceph-mon[25136]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:03.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:02 smithi027 ceph-mon[29814]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:03.173 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:02 smithi027 podman[63094]: 2024-06-21 00:54:02.889655481 +0000 UTC m=+0.142933806 container died 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, version=8, distribution-scope=public, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, release=754) 2024-06-21T00:54:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:02 smithi120 ceph-mon[25975]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:03.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 podman[63094]: 2024-06-21 00:54:03.249640895 +0000 UTC m=+0.502919210 container cleanup 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, release=754, architecture=x86_64, com.redhat.component=centos-stream-container, name=centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-06-21T00:54:03.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 bash[63094]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y 2024-06-21T00:54:03.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:03.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 podman[63107]: 2024-06-21 00:54:03.559938263 +0000 UTC m=+0.669562866 container remove 53e9dc04bcf4614d36ab9d93a86573d0c5c5dc4f24698dc186f1d543ee2e1ff7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, release=754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-06-21T00:54:03.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-06-21T00:54:03.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-06-21T00:54:03.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 systemd[1]: Stopped Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:54:03.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Consumed 1min 3.982s CPU time. 2024-06-21T00:54:04.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:03 smithi027 systemd[1]: Starting Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:54:04.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 podman[63236]: 2024-06-21 00:54:04.072947594 +0000 UTC m=+0.018213552 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T00:54:04.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 podman[63236]: 2024-06-21 00:54:04.177620516 +0000 UTC m=+0.122886463 container create 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 podman[63236]: 2024-06-21 00:54:04.421820905 +0000 UTC m=+0.367086864 container init 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 podman[63236]: 2024-06-21 00:54:04.424755306 +0000 UTC m=+0.370021254 container start 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 ceph-mgr[63256]: -- 172.21.15.27:0/4083652926 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555ca3e1d0e0 con 0x555ca3dfb400 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 bash[63236]: 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 systemd[1]: Started Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:54:04.557 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:04.557+0000 7efdef063040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:54:04.967 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:04.725+0000 7efdef063040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:54:04.967 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:04 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:04.799+0000 7efdef063040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:54:05.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:05.072+0000 7efdef063040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:54:05.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:05.139+0000 7efdef063040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:54:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.894 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:05.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:06.423 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.181+0000 7efdef063040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:54:06.424 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.307+0000 7efdef063040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:54:06.424 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.369+0000 7efdef063040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:54:06.697 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.606+0000 7efdef063040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:54:06.698 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.672+0000 7efdef063040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:54:06.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-mon[25136]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:06.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-mon[29814]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:06.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.749+0000 7efdef063040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:54:06.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:06.907+0000 7efdef063040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:54:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:06 smithi120 ceph-mon[25975]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T00:54:07.543 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:07.366+0000 7efdef063040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:54:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.867 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:54:07.868 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.869 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.870 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:07.871 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:54:07.587+0000 7f39c9fce640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.872 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:54:07.873 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.wait_async( 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return future.result(timeout) 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.__get_result() 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise self._exception 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: out, err, code = await self._run_cephadm( 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:54:07.874 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: raise OrchestratorError( 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Creating ceph-iscsi config... 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: exec(code, run_globals) 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.875 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.876 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.960 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.961 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T00:54:07.962 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.963 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.964 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T00:54:07.965 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.966 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.967 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.968 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T00:54:07.969 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:07.970 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:07.717+0000 7efdef063040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:54:07.970 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:07.791+0000 7efdef063040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:54:07.970 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:07.961+0000 7efdef063040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T00:54:08.439 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:08] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:54:08.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.316+0000 7efdef063040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:54:08.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.385+0000 7efdef063040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:54:08.717 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:08.915 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-mon[25136]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:08.916 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-mon[29814]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:08.916 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.586+0000 7efdef063040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:54:08.916 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.756+0000 7efdef063040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:54:08.916 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.825+0000 7efdef063040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:54:08.916 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.917+0000 7efdef063040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:54:09.048 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:08 smithi120 ceph-mon[25975]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:09.200 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:08.983+0000 7efdef063040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: Standby manager daemon y restarted 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: Standby manager daemon y started 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:54:09.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: Standby manager daemon y restarted 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: Standby manager daemon y started 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:09.596+0000 7efdef063040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:09.783+0000 7efdef063040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:54:09] ENGINE Bus STARTING 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: CherryPy Checker: 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: The Application mounted at '' has an empty config. 2024-06-21T00:54:09.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:54:09.891 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:54:09] ENGINE Serving on http://:::9283 2024-06-21T00:54:09.891 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:54:09 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:54:09] ENGINE Bus STARTED 2024-06-21T00:54:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: Standby manager daemon y restarted 2024-06-21T00:54:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: Standby manager daemon y started 2024-06-21T00:54:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T00:54:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:54:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T00:54:10.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:09 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:54:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:10 smithi027 ceph-mon[25136]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-21T00:54:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:10 smithi027 ceph-mon[25136]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:10 smithi027 ceph-mon[29814]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-21T00:54:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:10 smithi027 ceph-mon[29814]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:10 smithi120 ceph-mon[25975]: mgrmap e26: x(active, since 3m), standbys: y 2024-06-21T00:54:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:10 smithi120 ceph-mon[25975]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T00:54:12.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:12 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:54:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:12 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:13.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[25136]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 122 op/s 2024-06-21T00:54:13.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:13.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:13.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[25136]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-21T00:54:13.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[29814]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 122 op/s 2024-06-21T00:54:13.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:13.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:13.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:13 smithi027 ceph-mon[29814]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-21T00:54:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:13 smithi120 ceph-mon[25975]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 122 op/s 2024-06-21T00:54:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:13 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:54:13.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:13 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:54:13.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:13 smithi120 ceph-mon[25975]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-21T00:54:13.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1579893914' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:15.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:15 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1579893914' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:54:15.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:15.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:54:15.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:15 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1579893914' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:54:17.435 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:17 smithi027 ceph-mon[25136]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:17.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:17 smithi027 ceph-mon[29814]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:17.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:17 smithi120 ceph-mon[25975]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 522 B/s rd, 0 op/s 2024-06-21T00:54:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:18] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:54:19.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 261 B/s rd, 0 op/s 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 261 B/s rd, 0 op/s 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:19.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:19.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:19.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:19 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 261 B/s rd, 0 op/s 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:19 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:54:20.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[25136]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-21T00:54:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[25136]: Cluster is now healthy 2024-06-21T00:54:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[29814]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-21T00:54:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[29814]: Cluster is now healthy 2024-06-21T00:54:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:20 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:20.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:20 smithi120 ceph-mon[25975]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-06-21T00:54:20.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:20 smithi120 ceph-mon[25975]: Cluster is now healthy 2024-06-21T00:54:20.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:20 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:21 smithi027 ceph-mon[25136]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:21 smithi027 ceph-mon[29814]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:21 smithi120 ceph-mon[25975]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 291 B/s rd, 0 op/s 2024-06-21T00:54:23.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:23 smithi120 ceph-mon[25975]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:23.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:23 smithi027 ceph-mon[25136]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:23 smithi027 ceph-mon[29814]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:25 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:25 smithi120 ceph-mon[25975]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:25.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:25 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:25 smithi027 ceph-mon[29814]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:25 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:25 smithi027 ceph-mon[25136]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 663 B/s rd, 0 op/s 2024-06-21T00:54:26.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:26 smithi027 ceph-mon[29814]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:54:26.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:26 smithi027 ceph-mon[25136]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:54:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:26 smithi120 ceph-mon[25975]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:54:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:28] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-06-21T00:54:29.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:29 smithi120 ceph-mon[25975]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-21T00:54:29.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:29 smithi027 ceph-mon[25136]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-21T00:54:29.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:29 smithi027 ceph-mon[29814]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-06-21T00:54:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:31 smithi120 ceph-mon[25975]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:31 smithi027 ceph-mon[25136]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:31.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:31 smithi027 ceph-mon[29814]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:33 smithi120 ceph-mon[25975]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:33.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:33 smithi027 ceph-mon[25136]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:33.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:33 smithi027 ceph-mon[29814]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:35 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:35 smithi120 ceph-mon[25975]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:35.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:35.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[29814]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:35.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[25136]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:36.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:36 smithi027 ceph-mon[25136]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:36.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:36 smithi027 ceph-mon[29814]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:36 smithi120 ceph-mon[25975]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:38] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:54:39.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:39 smithi120 ceph-mon[25975]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:39 smithi027 ceph-mon[25136]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:39 smithi027 ceph-mon[29814]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:41.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:41 smithi120 ceph-mon[25975]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:41 smithi027 ceph-mon[25136]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:41.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:41 smithi027 ceph-mon[29814]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:43 smithi120 ceph-mon[25975]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:43.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:43 smithi027 ceph-mon[25136]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:43 smithi027 ceph-mon[29814]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:45 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:45.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:45 smithi120 ceph-mon[25975]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:45 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:45 smithi027 ceph-mon[25136]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:45 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:45 smithi027 ceph-mon[29814]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:46 smithi027 ceph-mon[25136]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:46 smithi027 ceph-mon[29814]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:46 smithi120 ceph-mon[25975]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:48] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:54:49.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:49 smithi120 ceph-mon[25975]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:49 smithi027 ceph-mon[25136]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:49.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:49 smithi027 ceph-mon[29814]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:50.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:54:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:51 smithi120 ceph-mon[25975]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:51 smithi027 ceph-mon[25136]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:51.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:51 smithi027 ceph-mon[29814]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:53 smithi027 ceph-mon[25136]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:53.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:53 smithi027 ceph-mon[29814]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:53.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:53 smithi120 ceph-mon[25975]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:53.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:53.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:55 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:55 smithi120 ceph-mon[25975]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:55 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:55 smithi027 ceph-mon[25136]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:55 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:54:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:55 smithi027 ceph-mon[29814]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:56 smithi027 ceph-mon[25136]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:56 smithi027 ceph-mon[29814]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:56 smithi120 ceph-mon[25975]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:54:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:54:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:54:58] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-06-21T00:54:59.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:54:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:54:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:54:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:54:59 smithi120 ceph-mon[25975]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:59.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:54:59 smithi027 ceph-mon[25136]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:54:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:54:59 smithi027 ceph-mon[29814]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:01 smithi120 ceph-mon[25975]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:01.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:01 smithi027 ceph-mon[25136]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:01.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:01 smithi027 ceph-mon[29814]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:03 smithi120 ceph-mon[25975]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:03.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:03 smithi027 ceph-mon[25136]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:03.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:03 smithi027 ceph-mon[29814]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:05 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:06.350 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:05 smithi120 ceph-mon[25975]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:06.350 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[25136]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[29814]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:06.351 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:06 smithi120 ceph-mon[25975]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:06 smithi027 ceph-mon[25136]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:06 smithi027 ceph-mon[29814]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:08] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:55:09.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:09 smithi120 ceph-mon[25975]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:09.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:09 smithi027 ceph-mon[25136]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:09 smithi027 ceph-mon[29814]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:11 smithi120 ceph-mon[25975]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:11 smithi027 ceph-mon[25136]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:11.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:11 smithi027 ceph-mon[29814]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:13 smithi120 ceph-mon[25975]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:13.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:13.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:13 smithi027 ceph-mon[25136]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:13.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:13 smithi027 ceph-mon[29814]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:15 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:15 smithi120 ceph-mon[25975]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:15.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:15 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:15.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:15 smithi027 ceph-mon[25136]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:15 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:15 smithi027 ceph-mon[29814]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:16 smithi120 ceph-mon[25975]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:16 smithi027 ceph-mon[25136]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:16 smithi027 ceph-mon[29814]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:18] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:55:19.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:19 smithi120 ceph-mon[25975]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:19 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:55:19.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:19 smithi027 ceph-mon[25136]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:19.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:19 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:55:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:19 smithi027 ceph-mon[29814]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:19 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:55:20.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:20 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:20.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:20 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:20.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:20 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:21 smithi120 ceph-mon[25975]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:21.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:21 smithi027 ceph-mon[25136]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:21.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:21 smithi027 ceph-mon[29814]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:23.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:55:23.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:23 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:55:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:23 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:55:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:25 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:25 smithi120 ceph-mon[25975]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:25.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:25 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:25 smithi027 ceph-mon[25136]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:25 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:25 smithi027 ceph-mon[29814]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:26 smithi120 ceph-mon[25975]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:26 smithi027 ceph-mon[25136]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:26 smithi027 ceph-mon[29814]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:28] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:55:29.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:29 smithi120 ceph-mon[25975]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:29.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:29 smithi027 ceph-mon[25136]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:29.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:29 smithi027 ceph-mon[29814]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:31 smithi120 ceph-mon[25975]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:31.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:31 smithi027 ceph-mon[25136]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:31.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:31 smithi027 ceph-mon[29814]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:33 smithi120 ceph-mon[25975]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:33.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:33 smithi027 ceph-mon[25136]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:33.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:33 smithi027 ceph-mon[29814]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:35 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:35 smithi120 ceph-mon[25975]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:35.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[25136]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[29814]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:36 smithi120 ceph-mon[25975]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:36 smithi027 ceph-mon[25136]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:37.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:36 smithi027 ceph-mon[29814]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:38] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-21T00:55:39.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:39 smithi120 ceph-mon[25975]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:39 smithi027 ceph-mon[25136]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:39 smithi027 ceph-mon[29814]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:41.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:41 smithi120 ceph-mon[25975]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:41 smithi027 ceph-mon[25136]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:41 smithi027 ceph-mon[29814]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:43 smithi120 ceph-mon[25975]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:43.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:43.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:43 smithi027 ceph-mon[25136]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:43 smithi027 ceph-mon[29814]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:45 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:45 smithi120 ceph-mon[25975]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:45 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:45 smithi027 ceph-mon[25136]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:45 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:45 smithi027 ceph-mon[29814]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:46 smithi120 ceph-mon[25975]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:47.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:46 smithi027 ceph-mon[25136]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:47.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:46 smithi027 ceph-mon[29814]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:48] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-21T00:55:49.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:49 smithi120 ceph-mon[25975]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:49 smithi027 ceph-mon[25136]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:49 smithi027 ceph-mon[29814]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:50.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:55:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:51 smithi120 ceph-mon[25975]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:51 smithi027 ceph-mon[25136]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:51.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:51 smithi027 ceph-mon[29814]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:53.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:53 smithi120 ceph-mon[25975]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:53.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:53.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:53 smithi027 ceph-mon[25136]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:53.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:53 smithi027 ceph-mon[29814]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:55 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:55 smithi120 ceph-mon[25975]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:55 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:55 smithi027 ceph-mon[25136]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:55 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:55:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:55 smithi027 ceph-mon[29814]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:56 smithi120 ceph-mon[25975]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:56 smithi027 ceph-mon[25136]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:56 smithi027 ceph-mon[29814]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:55:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:55:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:55:58] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-21T00:55:59.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:55:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:55:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"172.21.15.120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", cluster_addr=\"172.21.15.27\", device_class=\"ssd\", hostname=\"smithi027\", instance=\"smithi120:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.27\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:55:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:55:59 smithi120 ceph-mon[25975]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:59.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:55:59 smithi027 ceph-mon[25136]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:55:59.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:55:59 smithi027 ceph-mon[29814]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:01 smithi120 ceph-mon[25975]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:01.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:01 smithi027 ceph-mon[25136]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:01.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:01 smithi027 ceph-mon[29814]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:03 smithi120 ceph-mon[25975]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:03.867 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:56:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:56:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi027\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"47651940-2f66-11ef-bc9d-c7b262605968\", domainname=\"(none)\", instance=\"smithi027\", job=\"node\", machine=\"x86_64\", nodename=\"smithi027\", release=\"5.14.0-460.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-06-21T00:56:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:03 smithi027 ceph-mon[25136]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:03.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:03 smithi027 ceph-mon[29814]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:05 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:05 smithi120 ceph-mon[25975]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[25136]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[29814]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:06 smithi120 ceph-mon[25975]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:06 smithi027 ceph-mon[25136]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:06 smithi027 ceph-mon[29814]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:08] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:09.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:09 smithi120 ceph-mon[25975]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:09.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:09 smithi027 ceph-mon[25136]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:09 smithi027 ceph-mon[29814]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:11 smithi120 ceph-mon[25975]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:11 smithi027 ceph-mon[25136]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:11.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:11 smithi027 ceph-mon[29814]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:13 smithi120 ceph-mon[25975]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:13.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:13 smithi027 ceph-mon[25136]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:13.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:13 smithi027 ceph-mon[29814]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:15 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:15 smithi120 ceph-mon[25975]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:15 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:15 smithi027 ceph-mon[25136]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:15 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:15.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:15 smithi027 ceph-mon[29814]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:16 smithi120 ceph-mon[25975]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:16 smithi027 ceph-mon[25136]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:16 smithi027 ceph-mon[29814]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:18] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:19 smithi120 ceph-mon[25975]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:19.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:19 smithi027 ceph-mon[25136]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:19 smithi027 ceph-mon[29814]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:20.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:20 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:20.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:20 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:20.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:20 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:21 smithi120 ceph-mon[25975]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:21.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:21 smithi027 ceph-mon[25136]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:21.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:21 smithi027 ceph-mon[29814]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:23.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:23 smithi120 ceph-mon[25975]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:23.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:23 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:56:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:23 smithi027 ceph-mon[25136]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:23 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:56:23.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:23 smithi027 ceph-mon[29814]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:23 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:56:24.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:56:24.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:56:24.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[25136]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:56:24.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:56:24.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:56:24.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:24 smithi027 ceph-mon[29814]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:56:24.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:24 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:56:24.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:24 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:56:24.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:24 smithi120 ceph-mon[25975]: from='mgr.34799 ' entity='mgr.x' 2024-06-21T00:56:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:25 smithi120 ceph-mon[25975]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:25.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:25 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:25.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:25 smithi027 ceph-mon[29814]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:25.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:25 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:25 smithi027 ceph-mon[25136]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:25.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:25 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:26 smithi120 ceph-mon[25975]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:26 smithi027 ceph-mon[25136]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:27.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:26 smithi027 ceph-mon[29814]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:28] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:29 smithi120 ceph-mon[25975]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:29.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:29 smithi027 ceph-mon[25136]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:29.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:29 smithi027 ceph-mon[29814]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:31.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:31 smithi120 ceph-mon[25975]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:31.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:31 smithi027 ceph-mon[25136]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:31.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:31 smithi027 ceph-mon[29814]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:33 smithi120 ceph-mon[25975]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:33 smithi027 ceph-mon[25136]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:33.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:33 smithi027 ceph-mon[29814]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:35 smithi120 ceph-mon[25975]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:35 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:35.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:35 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[25136]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:35.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:35.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[29814]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:35 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:36 smithi120 ceph-mon[25975]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:36 smithi027 ceph-mon[25136]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:36 smithi027 ceph-mon[29814]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:38] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:39 smithi120 ceph-mon[25975]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:39 smithi027 ceph-mon[25136]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:39 smithi027 ceph-mon[29814]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:41.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:41 smithi120 ceph-mon[25975]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:41 smithi027 ceph-mon[25136]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:41 smithi027 ceph-mon[29814]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:43 smithi120 ceph-mon[25975]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:43 smithi027 ceph-mon[25136]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:43 smithi027 ceph-mon[29814]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:45 smithi120 ceph-mon[25975]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:45 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:45 smithi027 ceph-mon[25136]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:45 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:45 smithi027 ceph-mon[29814]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:45.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:45 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:46 smithi120 ceph-mon[25975]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:47.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:46 smithi027 ceph-mon[25136]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:47.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:46 smithi027 ceph-mon[29814]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:48] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:49 smithi120 ceph-mon[25975]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:49 smithi027 ceph-mon[25136]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:49.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:49 smithi027 ceph-mon[29814]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:50.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:50 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:50 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:50 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:56:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:51 smithi120 ceph-mon[25975]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:51 smithi027 ceph-mon[25136]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:51.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:51 smithi027 ceph-mon[29814]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:53.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:53 smithi120 ceph-mon[25975]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:53.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:53 smithi027 ceph-mon[25136]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:53.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:53 smithi027 ceph-mon[29814]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:55 smithi120 ceph-mon[25975]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:55 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:55 smithi027 ceph-mon[25136]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:55 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:55 smithi027 ceph-mon[29814]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:55.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:55 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:56:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:56 smithi120 ceph-mon[25975]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:56 smithi027 ceph-mon[25136]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:57.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:56 smithi027 ceph-mon[29814]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:56:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:56:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:56:58] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-06-21T00:56:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:56:59 smithi120 ceph-mon[25975]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:59.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:56:59 smithi027 ceph-mon[25136]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:56:59.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:56:59 smithi027 ceph-mon[29814]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:01 smithi120 ceph-mon[25975]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:01.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:01 smithi027 ceph-mon[25136]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:01.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:01 smithi027 ceph-mon[29814]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:03.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T00:57:03.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:03 smithi027 ceph-mon[25136]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:03.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:03 smithi027 ceph-mon[29814]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:03 smithi120 ceph-mon[25975]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (6m) 102s ago 13m 22.6M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (5m) 2m ago 13m 54.8M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 102s ago 12m 46.4M - 3.5 e1d6a67b021e a9d63ac90589 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (8m) 2m ago 18m 503M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T00:57:05.665 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (3m) 102s ago 20m 440M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (20m) 102s ago 20m 57.2M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (18m) 2m ago 18m 48.6M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (18m) 102s ago 18m 47.7M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 102s ago 14m 16.4M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (5m) 2m ago 14m 16.6M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (18m) 102s ago 18m 53.0M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (17m) 102s ago 17m 56.5M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (17m) 102s ago 17m 48.5M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (16m) 102s ago 16m 53.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (16m) 2m ago 16m 53.4M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (16m) 2m ago 16m 52.5M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (15m) 2m ago 15m 49.7M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T00:57:05.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (15m) 2m ago 15m 52.5M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T00:57:05.667 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (5m) 2m ago 14m 71.1M - 2.33.4 514e6a882f6e b952bfcfeb28 2024-06-21T00:57:05.667 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (13m) 102s ago 13m 76.1M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T00:57:05.667 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (13m) 2m ago 13m 73.5M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T00:57:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:05 smithi120 ceph-mon[25975]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:05 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:05.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:05 smithi120 ceph-mon[25975]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[25136]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:05.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[25136]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[29814]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:05.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:05 smithi027 ceph-mon[29814]: from='mgr.34799 172.21.15.120:0/2420864188' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:06.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T00:57:06.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:06 smithi027 ceph-mon[25136]: from='client.15252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:57:06.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:06 smithi027 ceph-mon[25136]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:06.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:06 smithi027 ceph-mon[29814]: from='client.15252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:57:06.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:06 smithi027 ceph-mon[29814]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:07.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:06 smithi120 ceph-mon[25975]: from='client.15252 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T00:57:07.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:06 smithi120 ceph-mon[25975]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T00:57:08.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T00:57:08.506 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T00:57:08.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:08 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1780970480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:57:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:57:08] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-21T00:57:08.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:08 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1780970480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:57:08.814 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:08 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1780970480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T00:57:09.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T00:57:09.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:09 smithi027 ceph-mon[25136]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:09.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:09 smithi027 ceph-mon[29814]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:09 smithi120 ceph-mon[25975]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:11.066 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T00:57:11.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:11 smithi027 ceph-mon[25136]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:11.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:11 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2872831546' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:57:11.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:11 smithi027 ceph-mon[29814]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:11.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:11 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2872831546' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:57:11.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph -s' 2024-06-21T00:57:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:11 smithi120 ceph-mon[25975]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:11 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2872831546' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: id: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-06-21T00:57:13.811 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: x(active, since 6m), standbys: y 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 241 objects, 585 KiB 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-06-21T00:57:13.812 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T00:57:13.824 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:13 smithi027 ceph-mon[25136]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:13.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:13 smithi027 ceph-mon[29814]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:13 smithi120 ceph-mon[25975]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:14.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph mgr fail' 2024-06-21T00:57:14.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:14 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2713888169' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:57:14.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:14 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2713888169' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:57:14.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:14 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2713888169' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T00:57:15.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:15 smithi027 ceph-mon[29814]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:15.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:15 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:15.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:15 smithi027 ceph-mon[25136]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:15.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:15 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:15 smithi120 ceph-mon[25975]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:15.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:15 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:17.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[25136]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:17.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:57:17.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[25136]: osdmap e104: 8 total, 8 up, 8 in 2024-06-21T00:57:17.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[29814]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:17.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:57:17.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-mon[29814]: osdmap e104: 8 total, 8 up, 8 in 2024-06-21T00:57:17.075 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:16 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:16] ENGINE Bus STOPPING 2024-06-21T00:57:17.076 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-mon[25975]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:17.076 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T00:57:17.076 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-mon[25975]: osdmap e104: 8 total, 8 up, 8 in 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:16.776+0000 7f3a34a01640 -1 mgr handle_mgr_map I was active but no longer am 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ignoring --setuser ceph since I am not root 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ignoring --setgroup ceph since I am not root 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-mgr[46008]: -- 172.21.15.120:0/263909992 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f7e66cd0e0 con 0x55f7e66ab400 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:16.930+0000 7f7a7eb42040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T00:57:17.077 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:17.078+0000 7f7a7eb42040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:57:17.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:17.148+0000 7f7a7eb42040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T00:57:17.391 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T00:57:17.392 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:17] ENGINE Bus STOPPED 2024-06-21T00:57:17.392 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:17] ENGINE Bus STARTING 2024-06-21T00:57:17.392 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:17] ENGINE Serving on http://:::9283 2024-06-21T00:57:17.392 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:00:57:17] ENGINE Bus STARTED 2024-06-21T00:57:17.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T00:57:17.769 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:17.394+0000 7f7a7eb42040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T00:57:17.769 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:17.455+0000 7f7a7eb42040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: mgrmap e27: y(active, starting, since 0.218596s) 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:57:18.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: Manager daemon y is now available 2024-06-21T00:57:18.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: mgrmap e27: y(active, starting, since 0.218596s) 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:57:18.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T00:57:18.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:18.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:18.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:17 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2551688976' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T00:57:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: mgrmap e27: y(active, starting, since 0.218596s) 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T00:57:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: Manager daemon y is now available 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:17 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T00:57:18.784 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:18.471+0000 7f7a7eb42040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T00:57:18.784 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:18.589+0000 7f7a7eb42040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T00:57:18.784 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:18.655+0000 7f7a7eb42040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T00:57:19.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:18 smithi027 ceph-mon[25136]: mgrmap e28: y(active, since 1.22814s) 2024-06-21T00:57:19.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:18 smithi027 ceph-mon[25136]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:19.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:18 smithi027 ceph-mon[29814]: mgrmap e28: y(active, since 1.22814s) 2024-06-21T00:57:19.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:18 smithi027 ceph-mon[29814]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:19.064 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-mon[25975]: mgrmap e28: y(active, since 1.22814s) 2024-06-21T00:57:19.064 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-mon[25975]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:19.064 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:18.885+0000 7f7a7eb42040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T00:57:19.064 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:18.950+0000 7f7a7eb42040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T00:57:19.065 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:19.027+0000 7f7a7eb42040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T00:57:19.306 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:57:19.055+0000 7efd4c35e640 -1 log_channel(cephadm) log [ERR] : [21/Jun/2024:00:57:19] ENGINE Error in HTTPServer.serve 2024-06-21T00:57:19.306 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._connections.run(self.expiration_interval) 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._run(expiration_interval) 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s = self.context.wrap_socket( 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.sslsocket_class._create( 2024-06-21T00:57:19.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:57:19.308 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self.do_handshake() 2024-06-21T00:57:19.308 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:57:19.308 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._sslobj.do_handshake() 2024-06-21T00:57:19.308 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:57:19.308 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T00:57:19.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:19.182+0000 7f7a7eb42040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T00:57:19.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: mgrmap e29: y(active, since 2s) 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: [21/Jun/2024:00:57:18] ENGINE Bus STARTING 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: [21/Jun/2024:00:57:19] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: [21/Jun/2024:00:57:19] ENGINE Error in HTTPServer.serve 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:57:19.890 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: [21/Jun/2024:00:57:19] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[25136]: [21/Jun/2024:00:57:19] ENGINE Bus STARTED 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: mgrmap e29: y(active, since 2s) 2024-06-21T00:57:19.891 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: [21/Jun/2024:00:57:18] ENGINE Bus STARTING 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: [21/Jun/2024:00:57:19] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: [21/Jun/2024:00:57:19] ENGINE Error in HTTPServer.serve 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T00:57:19.892 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: [21/Jun/2024:00:57:19] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T00:57:19.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:19 smithi027 ceph-mon[29814]: [21/Jun/2024:00:57:19] ENGINE Bus STARTED 2024-06-21T00:57:19.971 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:19.649+0000 7f7a7eb42040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: mgrmap e29: y(active, since 2s) 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: [21/Jun/2024:00:57:18] ENGINE Bus STARTING 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: [21/Jun/2024:00:57:19] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: [21/Jun/2024:00:57:19] ENGINE Error in HTTPServer.serve 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._connections.run(self.expiration_interval) 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._run(expiration_interval) 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T00:57:19.972 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: s = self.context.wrap_socket( 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.sslsocket_class._create( 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.do_handshake() 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._sslobj.do_handshake() 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: [21/Jun/2024:00:57:19] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T00:57:19.973 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-mon[25975]: [21/Jun/2024:00:57:19] ENGINE Bus STARTED 2024-06-21T00:57:20.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:19.989+0000 7f7a7eb42040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T00:57:20.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.057+0000 7f7a7eb42040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T00:57:20.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.210+0000 7f7a7eb42040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T00:57:20.799 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.542+0000 7f7a7eb42040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T00:57:20.799 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.606+0000 7f7a7eb42040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T00:57:21.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.799+0000 7f7a7eb42040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T00:57:21.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:20.954+0000 7f7a7eb42040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T00:57:21.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:21.031+0000 7f7a7eb42040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T00:57:21.402 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:21.119+0000 7f7a7eb42040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T00:57:21.402 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:21.185+0000 7f7a7eb42040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T00:57:21.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[25136]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:21.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[25136]: mgrmap e30: y(active, since 4s) 2024-06-21T00:57:21.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:21.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:21.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[29814]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:21.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[29814]: mgrmap e30: y(active, since 4s) 2024-06-21T00:57:21.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:21.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:21 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-mon[25975]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-mon[25975]: mgrmap e30: y(active, since 4s) 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:21.753+0000 7f7a7eb42040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T00:57:22.031 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:21.926+0000 7f7a7eb42040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:57:21] ENGINE Bus STARTING 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: CherryPy Checker: 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: The Application mounted at '' has an empty config. 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:57:22] ENGINE Serving on http://:::9283 2024-06-21T00:57:22.032 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:00:57:22] ENGINE Bus STARTED 2024-06-21T00:57:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T00:57:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:57:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:57:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:57:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:57:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:23.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:22 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:23.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:22 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:23 smithi120 ceph-mon[25975]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:23 smithi120 ceph-mon[25975]: mgrmap e31: y(active, since 6s), standbys: x 2024-06-21T00:57:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:23 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:57:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:23 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:23 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[25136]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[25136]: mgrmap e31: y(active, since 6s), standbys: x 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[29814]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T00:57:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[29814]: mgrmap e31: y(active, since 6s), standbys: x 2024-06-21T00:57:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T00:57:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:23 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:24 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T00:57:25.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:25.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:24 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:25 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: from='client.15240 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:26.121 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:26.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:25 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T00:57:27.947 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:27 smithi027 ceph-mon[25136]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:57:27.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:27 smithi027 ceph-mon[29814]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:57:28.347 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:27 smithi120 ceph-mon[25975]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T00:57:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 00:57:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:57:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T00:57:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:57:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:57:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:57:30.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:29 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1884673016' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:57:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:57:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1884673016' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T00:57:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:29 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1884673016' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T00:57:32.077 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:31 smithi120 ceph-mon[25975]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:32.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:31 smithi027 ceph-mon[25136]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:31 smithi027 ceph-mon[29814]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:32.365 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:57:32.365 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 bash[54322]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:57:32.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T00:57:32.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T00:57:32.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.367Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.368Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.368Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[49094]: ts=2024-06-21T00:57:32.368Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T00:57:32.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 podman[54332]: 2024-06-21 00:57:32.377645125 +0000 UTC m=+0.133090844 container died b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:32 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:33.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 podman[54332]: 2024-06-21 00:57:32.700347917 +0000 UTC m=+0.455793631 container cleanup b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 bash[54332]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T00:57:33.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 bash[54359]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T00:57:33.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 podman[54346]: 2024-06-21 00:57:32.901836009 +0000 UTC m=+0.525246732 container remove b952bfcfeb2885e8efaf005b3dd3d08e185c771175de073a2ebd3f6178458ee2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T00:57:33.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:57:33.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 2.021s CPU time. 2024-06-21T00:57:33.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:32 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T00:57:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:32 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:32 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:33.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 podman[54421]: 2024-06-21 00:57:33.043037172 +0000 UTC m=+0.018880470 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-21T00:57:33.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 podman[54421]: 2024-06-21 00:57:33.147856792 +0000 UTC m=+0.123700121 container create 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 podman[54421]: 2024-06-21 00:57:33.371777453 +0000 UTC m=+0.347620752 container init 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.396 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 podman[54421]: 2024-06-21 00:57:33.374426068 +0000 UTC m=+0.350269364 container start 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.398Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.400Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T00:57:33.770 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.400Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.401Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.404Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.404Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.682µs 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.404Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 bash[54421]: 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T00:57:33.771 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.610Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-06-21T00:57:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:33.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:33 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:34.067 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:33.847Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-06-21T00:57:34.067 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:34.067 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:34.067 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:34.067 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:34.068 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:33 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:57:34.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.069Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-06-21T00:57:34.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.069Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-06-21T00:57:34.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.069Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.236µs wal_replay_duration=665.212207ms total_replay_duration=665.258264ms 2024-06-21T00:57:34.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.073Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T00:57:34.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.073Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T00:57:34.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.074Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T00:57:34.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.092Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.544508ms db_storage=716ns remote_storage=1.3µs web_handler=319ns query_engine=634ns scrape=358.5µs scrape_sd=94.171µs notify=11.304µs notify_sd=10.901µs rules=17.762788ms 2024-06-21T00:57:34.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T00:57:34.092Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T00:57:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-mon[25975]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:57:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:34.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:34 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:35.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:35.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[25136]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[29814]: Adding iSCSI gateway http://:@172.21.15.27:5000 to Dashboard 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi027"}]: dispatch 2024-06-21T00:57:35.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:34 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T00:57:36.015 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:35 smithi120 ceph-mon[25975]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:36.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:35 smithi027 ceph-mon[25136]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:36.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:35 smithi027 ceph-mon[29814]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:38.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:37 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:57:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:57:38.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:57:38.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:37 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:57:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:39 smithi120 ceph-mon[25975]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:57:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:39 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:39 smithi027 ceph-mon[25136]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:57:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:39 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:40.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:39 smithi027 ceph-mon[29814]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T00:57:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:39 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:41 smithi120 ceph-mon[25975]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:41 smithi027 ceph-mon[25136]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:42.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:41 smithi027 ceph-mon[29814]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:43 smithi120 ceph-mon[25975]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:43 smithi027 ceph-mon[25136]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:43 smithi027 ceph-mon[29814]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T00:57:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:45 smithi120 ceph-mon[25975]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:45 smithi027 ceph-mon[25136]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:46.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:45 smithi027 ceph-mon[29814]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:47 smithi120 ceph-mon[25975]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:47 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:47 smithi027 ceph-mon[25136]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:47 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:47 smithi027 ceph-mon[29814]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:47 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:57:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:57:48] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-21T00:57:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:49 smithi120 ceph-mon[25975]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:50.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:49 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:49 smithi027 ceph-mon[25136]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:49 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:49 smithi027 ceph-mon[29814]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:49 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:57:52.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:51 smithi027 ceph-mon[25136]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:52.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:51 smithi027 ceph-mon[29814]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:51 smithi120 ceph-mon[25975]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:53 smithi120 ceph-mon[25975]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:54.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:53 smithi027 ceph-mon[25136]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:54.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:53 smithi027 ceph-mon[29814]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:55 smithi120 ceph-mon[25975]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:55 smithi027 ceph-mon[25136]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:56.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:55 smithi027 ceph-mon[29814]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:57:58.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:57 smithi120 ceph-mon[25975]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:57 smithi027 ceph-mon[25136]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:57 smithi027 ceph-mon[29814]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:57:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:57:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:57:58] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-21T00:58:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:59 smithi120 ceph-mon[25975]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:57:59 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:59 smithi027 ceph-mon[29814]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:57:59 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:59 smithi027 ceph-mon[25136]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:57:59 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:01 smithi120 ceph-mon[25975]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:01 smithi027 ceph-mon[25136]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:01 smithi027 ceph-mon[29814]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:02 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:02 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:02 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:03 smithi027 ceph-mon[25136]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:03.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:03 smithi027 ceph-mon[29814]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:03 smithi120 ceph-mon[25975]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:05 smithi120 ceph-mon[25975]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:05 smithi027 ceph-mon[25136]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:06.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:05 smithi027 ceph-mon[29814]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:07 smithi120 ceph-mon[25975]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:07 smithi027 ceph-mon[25136]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:07 smithi027 ceph-mon[29814]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:08] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-06-21T00:58:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:09 smithi120 ceph-mon[25975]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:09 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:09 smithi027 ceph-mon[25136]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:09 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:09 smithi027 ceph-mon[29814]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:09 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:11 smithi120 ceph-mon[25975]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:11 smithi027 ceph-mon[29814]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:12.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:11 smithi027 ceph-mon[25136]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:13 smithi120 ceph-mon[25975]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:13 smithi027 ceph-mon[25136]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:13 smithi027 ceph-mon[29814]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:15 smithi120 ceph-mon[25975]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:15 smithi027 ceph-mon[25136]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:15 smithi027 ceph-mon[29814]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:17 smithi120 ceph-mon[25975]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:18.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:17 smithi027 ceph-mon[25136]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:17 smithi027 ceph-mon[29814]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:18] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T00:58:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:19 smithi120 ceph-mon[25975]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:19 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:19 smithi027 ceph-mon[25136]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:19 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:19 smithi027 ceph-mon[29814]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:19 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:22.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:21 smithi027 ceph-mon[25136]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:22.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:21 smithi027 ceph-mon[29814]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:21 smithi120 ceph-mon[25975]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:23 smithi120 ceph-mon[25975]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:23 smithi027 ceph-mon[25136]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:23 smithi027 ceph-mon[29814]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:25 smithi120 ceph-mon[25975]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:25 smithi027 ceph-mon[25136]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:26.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:25 smithi027 ceph-mon[29814]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:28.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:27 smithi120 ceph-mon[25975]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:27 smithi027 ceph-mon[25136]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:27 smithi027 ceph-mon[29814]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:28] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T00:58:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:29 smithi120 ceph-mon[25975]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:29 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:29 smithi027 ceph-mon[29814]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:29 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:29 smithi027 ceph-mon[25136]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:29 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:31 smithi120 ceph-mon[25975]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:31 smithi027 ceph-mon[29814]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:32.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:31 smithi027 ceph-mon[25136]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:32 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:32 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:32 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:33 smithi120 ceph-mon[25975]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:33 smithi027 ceph-mon[25136]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:33 smithi027 ceph-mon[29814]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:36.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:35 smithi120 ceph-mon[25975]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:35 smithi027 ceph-mon[25136]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:36.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:35 smithi027 ceph-mon[29814]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:37 smithi120 ceph-mon[25975]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:37 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:58:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:37 smithi027 ceph-mon[25136]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:37 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:58:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:37 smithi027 ceph-mon[29814]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:37 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:58:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:38] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-06-21T00:58:39.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:58:39.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:39.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:58:39.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:58:39.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:58:40.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:40 smithi027 ceph-mon[25136]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:40.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:40 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:40.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:40 smithi027 ceph-mon[29814]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:40.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:40 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:40.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:40 smithi120 ceph-mon[25975]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:40.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:40 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:41 smithi120 ceph-mon[25975]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:41 smithi027 ceph-mon[25136]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:41 smithi027 ceph-mon[29814]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:43 smithi120 ceph-mon[25975]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:43 smithi027 ceph-mon[25136]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:44.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:43 smithi027 ceph-mon[29814]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:45 smithi120 ceph-mon[25975]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:45 smithi027 ceph-mon[25136]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:46.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:45 smithi027 ceph-mon[29814]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:47 smithi027 ceph-mon[25136]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:47 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:47 smithi027 ceph-mon[29814]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:47 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:48.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:47 smithi120 ceph-mon[25975]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:48.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:47 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:58:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:48] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-06-21T00:58:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:49 smithi027 ceph-mon[25136]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:49 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:49 smithi027 ceph-mon[29814]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:49 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:50.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:49 smithi120 ceph-mon[25975]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:50.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:49 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:58:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:51 smithi120 ceph-mon[25975]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:51 smithi027 ceph-mon[25136]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:51 smithi027 ceph-mon[29814]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:53 smithi120 ceph-mon[25975]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:53 smithi027 ceph-mon[25136]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:53 smithi027 ceph-mon[29814]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:58:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:55 smithi120 ceph-mon[25975]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:55 smithi027 ceph-mon[25136]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:56.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:55 smithi027 ceph-mon[29814]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:57 smithi027 ceph-mon[25136]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:57 smithi027 ceph-mon[29814]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:58.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:57 smithi120 ceph-mon[25975]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:58:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:58:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:58:58] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-06-21T00:59:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:59 smithi027 ceph-mon[25136]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:58:59 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:59 smithi027 ceph-mon[29814]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:58:59 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:59 smithi120 ceph-mon[25975]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:58:59 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:01 smithi120 ceph-mon[25975]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:01 smithi027 ceph-mon[25136]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:01 smithi027 ceph-mon[29814]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:02 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:02 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:03.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:02 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:03 smithi120 ceph-mon[25975]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:03 smithi027 ceph-mon[25136]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:04.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:03 smithi027 ceph-mon[29814]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:05 smithi120 ceph-mon[25975]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:06.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:05 smithi027 ceph-mon[29814]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:05 smithi027 ceph-mon[25136]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:07 smithi027 ceph-mon[25136]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:07 smithi027 ceph-mon[29814]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:08.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:07 smithi120 ceph-mon[25975]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:08] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-06-21T00:59:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:09 smithi027 ceph-mon[25136]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:09 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:09 smithi027 ceph-mon[29814]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:09 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:09 smithi120 ceph-mon[25975]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:10.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:09 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:11 smithi120 ceph-mon[25975]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:11 smithi027 ceph-mon[25136]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:11 smithi027 ceph-mon[29814]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:13 smithi027 ceph-mon[25136]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:13 smithi027 ceph-mon[29814]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:14.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:13 smithi120 ceph-mon[25975]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:15 smithi027 ceph-mon[25136]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:15 smithi027 ceph-mon[29814]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:15 smithi120 ceph-mon[25975]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:17 smithi027 ceph-mon[25136]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:17 smithi027 ceph-mon[29814]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:17 smithi120 ceph-mon[25975]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:18] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:59:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:19 smithi027 ceph-mon[25136]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:19 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:19 smithi027 ceph-mon[29814]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:19 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:19 smithi120 ceph-mon[25975]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:19 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:21 smithi120 ceph-mon[25975]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:22.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:21 smithi027 ceph-mon[25136]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:22.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:21 smithi027 ceph-mon[29814]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:23 smithi027 ceph-mon[25136]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:23 smithi027 ceph-mon[29814]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:23 smithi120 ceph-mon[25975]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:25 smithi027 ceph-mon[25136]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:25 smithi027 ceph-mon[29814]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:25 smithi120 ceph-mon[25975]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:27 smithi027 ceph-mon[25136]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:27 smithi027 ceph-mon[29814]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:27 smithi120 ceph-mon[25975]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:28] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-06-21T00:59:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:29 smithi027 ceph-mon[25136]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:29 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:29 smithi027 ceph-mon[29814]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:29 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:29 smithi120 ceph-mon[25975]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:29 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:31 smithi120 ceph-mon[25975]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:32.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:31 smithi027 ceph-mon[25136]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:31 smithi027 ceph-mon[29814]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:32 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:32 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:32 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:34.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:33 smithi120 ceph-mon[25975]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:33 smithi027 ceph-mon[25136]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:33 smithi027 ceph-mon[29814]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:35 smithi027 ceph-mon[25136]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:36.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:35 smithi027 ceph-mon[29814]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:35 smithi120 ceph-mon[25975]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:37 smithi027 ceph-mon[25136]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:37 smithi027 ceph-mon[29814]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:38.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:37 smithi120 ceph-mon[25975]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:38.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:38 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:59:38.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:38 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:59:38.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:38] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T00:59:39.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:38 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T00:59:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[25136]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[25136]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[29814]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:59:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[29814]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:59:40.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:39 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:39 smithi120 ceph-mon[25975]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:39 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T00:59:40.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:39 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T00:59:40.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:39 smithi120 ceph-mon[25975]: from='mgr.15225 ' entity='mgr.y' 2024-06-21T00:59:40.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:39 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:41 smithi120 ceph-mon[25975]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:41 smithi027 ceph-mon[25136]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:42.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:41 smithi027 ceph-mon[29814]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:43 smithi027 ceph-mon[25136]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:43 smithi027 ceph-mon[29814]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:44.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:43 smithi120 ceph-mon[25975]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:45 smithi027 ceph-mon[25136]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:46.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:45 smithi027 ceph-mon[29814]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:45 smithi120 ceph-mon[25975]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:47 smithi027 ceph-mon[25136]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:47 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:47 smithi027 ceph-mon[29814]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:47 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:47 smithi120 ceph-mon[25975]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:47 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T00:59:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:48] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T00:59:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:49 smithi027 ceph-mon[25136]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:49 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:49 smithi027 ceph-mon[29814]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:49 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:50.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:49 smithi120 ceph-mon[25975]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:50.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:49 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T00:59:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:51 smithi120 ceph-mon[25975]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:51 smithi027 ceph-mon[25136]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:51 smithi027 ceph-mon[29814]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:53 smithi027 ceph-mon[25136]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:53 smithi027 ceph-mon[29814]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:53 smithi120 ceph-mon[25975]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T00:59:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:55 smithi027 ceph-mon[25136]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:56.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:55 smithi027 ceph-mon[29814]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:56.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:55 smithi120 ceph-mon[25975]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:57 smithi027 ceph-mon[25136]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:57 smithi027 ceph-mon[29814]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:58.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:57 smithi120 ceph-mon[25975]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T00:59:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 00:59:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:00:59:58] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:00:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:59 smithi027 ceph-mon[25136]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 00:59:59 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:59 smithi027 ceph-mon[29814]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 00:59:59 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:59 smithi120 ceph-mon[25975]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 00:59:59 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:01.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:00 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T01:00:01.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:00 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T01:00:01.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:00 smithi120 ceph-mon[25975]: overall HEALTH_OK 2024-06-21T01:00:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:01 smithi027 ceph-mon[25136]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:01 smithi027 ceph-mon[29814]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:02.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:01 smithi120 ceph-mon[25975]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:02 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:02 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:02 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:03 smithi027 ceph-mon[25136]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:03.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:03 smithi027 ceph-mon[29814]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:03 smithi120 ceph-mon[25975]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:05 smithi027 ceph-mon[25136]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:06.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:05 smithi027 ceph-mon[29814]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:06.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:05 smithi120 ceph-mon[25975]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:07 smithi027 ceph-mon[25136]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:07 smithi027 ceph-mon[29814]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:08.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:07 smithi120 ceph-mon[25975]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:00:08] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-06-21T01:00:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:09 smithi027 ceph-mon[25136]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:09 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:09 smithi027 ceph-mon[29814]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:09 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:09 smithi120 ceph-mon[25975]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:09 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:11 smithi120 ceph-mon[25975]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:11 smithi027 ceph-mon[25136]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:11 smithi027 ceph-mon[29814]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:13 smithi027 ceph-mon[25136]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:13 smithi027 ceph-mon[29814]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:14.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:13 smithi120 ceph-mon[25975]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:15 smithi027 ceph-mon[25136]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:15 smithi027 ceph-mon[29814]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:15 smithi120 ceph-mon[25975]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:17 smithi027 ceph-mon[25136]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:17 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:17 smithi027 ceph-mon[29814]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:17 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:17 smithi120 ceph-mon[25975]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:17 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:00:18] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:00:20.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:00:20.161 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:19 smithi027 ceph-mon[25136]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:20.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:19 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:20.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:19 smithi027 ceph-mon[29814]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:20.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:19 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:19 smithi120 ceph-mon[25975]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:19 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:21 smithi120 ceph-mon[25975]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:22.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:21 smithi027 ceph-mon[25136]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:22.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:21 smithi027 ceph-mon[29814]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 2m ago 17m 22.8M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (9m) 2m ago 16m 54.7M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 2m ago 16m 50.3M - 3.5 e1d6a67b021e 8424899de81a 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (11m) 2m ago 21m 437M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (6m) 2m ago 23m 491M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:00:22.258 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (23m) 2m ago 23m 62.0M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (22m) 2m ago 21m 49.4M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (22m) 2m ago 22m 51.9M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 17m 16.9M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (9m) 2m ago 17m 16.6M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (21m) 2m ago 21m 53.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (21m) 2m ago 21m 56.6M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (20m) 2m ago 20m 48.7M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (20m) 2m ago 20m 53.5M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (19m) 2m ago 19m 53.8M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (19m) 2m ago 19m 53.0M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (19m) 2m ago 19m 50.3M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (18m) 2m ago 18m 53.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 2m ago 18m 42.1M - 2.33.4 514e6a882f6e 9cf45c6f9e84 2024-06-21T01:00:22.259 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (16m) 2m ago 16m 77.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:00:22.260 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (16m) 2m ago 16m 75.3M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:00:22.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T01:00:23.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:22 smithi120 ceph-mon[25975]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:00:23.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:22 smithi027 ceph-mon[25136]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:00:23.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:22 smithi027 ceph-mon[29814]: from='client.25240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:00:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:23 smithi027 ceph-mon[25136]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:23.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:23 smithi027 ceph-mon[29814]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:23 smithi120 ceph-mon[25975]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:25.153 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:00:25.154 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:00:25.154 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:00:25.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:00:25.154 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:00:25.154 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:00:25.155 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:00:25.156 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:00:25.157 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:00:25.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph -s' 2024-06-21T01:00:26.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:25 smithi027 ceph-mon[25136]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:26.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:25 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2651476692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:26.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:25 smithi027 ceph-mon[29814]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:26.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:25 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2651476692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:25 smithi120 ceph-mon[25975]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:25 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2651476692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-06-21T01:00:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: id: 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 3m), standbys: x 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 241 objects, 585 KiB 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T01:00:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-06-21T01:00:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-06-21T01:00:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-21T01:00:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:27 smithi027 ceph-mon[25136]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:27 smithi027 ceph-mon[29814]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:27 smithi120 ceph-mon[25975]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:28.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:00:28.898 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:00:28] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:00:29.158 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:28 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1440879710' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T01:00:29.158 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:28 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1440879710' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T01:00:29.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:28 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1440879710' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-21T01:00:30.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:29 smithi027 ceph-mon[25136]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:30.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:29 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:30.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:29 smithi027 ceph-mon[29814]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:30.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:29 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:29 smithi120 ceph-mon[25975]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:29 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:30.782 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:00:31.086 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:30 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/485615057' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:00:31.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:30 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/485615057' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:00:31.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:30 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/485615057' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:00:31.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-06-21T01:00:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:31 smithi027 ceph-mon[25136]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:31.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:31 smithi027 ceph-mon[29814]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:31 smithi120 ceph-mon[25975]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:32 smithi027 ceph-mon[25136]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:32 smithi027 ceph-mon[29814]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:33.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:32 smithi120 ceph-mon[25975]: from='mgr.15225 172.21.15.27:0/3947241606' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:33.401 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:00:34.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:33 smithi027 ceph-mon[25136]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:34.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:33 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/143314395' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:34.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:33 smithi027 ceph-mon[29814]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:34.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:33 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/143314395' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:34.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph mgr fail' 2024-06-21T01:00:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:33 smithi120 ceph-mon[25975]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:00:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:33 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/143314395' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:00:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:35 smithi027 ceph-mon[25136]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:35 smithi027 ceph-mon[29814]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:35 smithi120 ceph-mon[25975]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2649211053' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[29814]: osdmap e105: 8 total, 8 up, 8 in 2024-06-21T01:00:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2649211053' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mon[25136]: osdmap e105: 8 total, 8 up, 8 in 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ignoring --setuser ceph since I am not root 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ignoring --setgroup ceph since I am not root 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:36 smithi027 ceph-mgr[63256]: -- 172.21.15.27:0/1063705434 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a71dfb50e0 con 0x55a71df93400 2024-06-21T01:00:37.229 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:37.099+0000 7fbe861e2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T01:00:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:36 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2649211053' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:36 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-06-21T01:00:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:36 smithi120 ceph-mon[25975]: osdmap e105: 8 total, 8 up, 8 in 2024-06-21T01:00:37.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:36 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:36] ENGINE Bus STOPPING 2024-06-21T01:00:37.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T01:00:37.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:37] ENGINE Bus STOPPED 2024-06-21T01:00:37.515 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:37.262+0000 7fbe861e2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:00:37.516 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:37.336+0000 7fbe861e2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T01:00:37.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 180' 2024-06-21T01:00:37.753 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:37] ENGINE Bus STARTING 2024-06-21T01:00:37.753 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:37] ENGINE Serving on http://:::9283 2024-06-21T01:00:37.753 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:00:37] ENGINE Bus STARTED 2024-06-21T01:00:37.873 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:37.605+0000 7fbe861e2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T01:00:37.874 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:37.674+0000 7fbe861e2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T01:00:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T01:00:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: mgrmap e32: x(active, starting, since 0.811992s) 2024-06-21T01:00:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:00:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:00:38.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: Manager daemon x is now available 2024-06-21T01:00:38.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:38.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:38.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:37 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: mgrmap e32: x(active, starting, since 0.811992s) 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:00:38.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: Manager daemon x is now available 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: mgrmap e32: x(active, starting, since 0.811992s) 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:00:38.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: Manager daemon x is now available 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:38.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:00:38.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:37 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-06-21T01:00:38.922 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:38.735+0000 7fbe861e2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T01:00:38.922 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:38.861+0000 7fbe861e2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T01:00:38.922 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:38.923+0000 7fbe861e2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T01:00:39.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[29814]: mgrmap e33: x(active, since 2s) 2024-06-21T01:00:39.212 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[29814]: [21/Jun/2024:01:00:38] ENGINE Bus STARTING 2024-06-21T01:00:39.212 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[29814]: [21/Jun/2024:01:00:38] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T01:00:39.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[25136]: mgrmap e33: x(active, since 2s) 2024-06-21T01:00:39.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[25136]: [21/Jun/2024:01:00:38] ENGINE Bus STARTING 2024-06-21T01:00:39.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-mon[25136]: [21/Jun/2024:01:00:38] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T01:00:39.213 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:39.165+0000 7fbe861e2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T01:00:39.272 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:39 smithi120 ceph-mon[25975]: mgrmap e33: x(active, since 2s) 2024-06-21T01:00:39.272 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:39 smithi120 ceph-mon[25975]: [21/Jun/2024:01:00:38] ENGINE Bus STARTING 2024-06-21T01:00:39.272 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:39 smithi120 ceph-mon[25975]: [21/Jun/2024:01:00:38] ENGINE Serving on http://172.21.15.120:8765 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:00:38.939+0000 7f79d6600640 -1 log_channel(cephadm) log [ERR] : [21/Jun/2024:01:00:38] ENGINE Error in HTTPServer.serve 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: Traceback (most recent call last): 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._connections.run(self.expiration_interval) 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._run(expiration_interval) 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:00:39.273 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: s = self.context.wrap_socket( 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: return self.sslsocket_class._create( 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self.do_handshake() 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: self._sslobj.do_handshake() 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:00:39.274 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:00:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:00:39.476 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:39.239+0000 7fbe861e2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T01:00:39.476 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:39.313+0000 7fbe861e2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T01:00:39.476 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:39.476+0000 7fbe861e2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:00:40.138 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:39.941+0000 7fbe861e2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[25136]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[25136]: [21/Jun/2024:01:00:38] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[25136]: [21/Jun/2024:01:00:38] ENGINE Bus STARTED 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[25136]: [21/Jun/2024:01:00:38] ENGINE Error in HTTPServer.serve 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:00:40.139 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:00:40.140 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:00:40.141 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:00:40.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[25136]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:40.209 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:40 smithi120 ceph-mon[25975]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:40.209 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:40 smithi120 ceph-mon[25975]: [21/Jun/2024:01:00:38] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:40 smithi120 ceph-mon[25975]: [21/Jun/2024:01:00:38] ENGINE Bus STARTED 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:40 smithi120 ceph-mon[25975]: [21/Jun/2024:01:00:38] ENGINE Error in HTTPServer.serve 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._run(expiration_interval) 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:00:40.210 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: s = self.context.wrap_socket( 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.sslsocket_class._create( 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.do_handshake() 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._sslobj.do_handshake() 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:00:40.211 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:40 smithi120 ceph-mon[25975]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:40.445 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[29814]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:40.445 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[29814]: [21/Jun/2024:01:00:38] ENGINE Serving on https://172.21.15.120:7150 2024-06-21T01:00:40.445 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[29814]: [21/Jun/2024:01:00:38] ENGINE Bus STARTED 2024-06-21T01:00:40.445 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[29814]: [21/Jun/2024:01:00:38] ENGINE Error in HTTPServer.serve 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:00:40.446 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-mon[29814]: from='client.15321 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:40.298+0000 7fbe861e2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T01:00:40.447 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:40.372+0000 7fbe861e2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T01:00:40.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:40.541+0000 7fbe861e2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T01:00:41.182 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-mon[25136]: mgrmap e34: x(active, since 4s) 2024-06-21T01:00:41.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-mon[29814]: mgrmap e34: x(active, since 4s) 2024-06-21T01:00:41.183 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:40.893+0000 7fbe861e2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T01:00:41.183 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:40.967+0000 7fbe861e2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T01:00:41.183 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:41.184+0000 7fbe861e2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T01:00:41.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:41.346+0000 7fbe861e2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T01:00:41.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:41.416+0000 7fbe861e2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T01:00:41.495 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:41 smithi120 ceph-mon[25975]: mgrmap e34: x(active, since 4s) 2024-06-21T01:00:41.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:41.507+0000 7fbe861e2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T01:00:41.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:41.571+0000 7fbe861e2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T01:00:42.338 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:42.167+0000 7fbe861e2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T01:00:42.338 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:42.340+0000 7fbe861e2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:00:42] ENGINE Bus STARTING 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: CherryPy Checker: 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: The Application mounted at '' has an empty config. 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:00:42] ENGINE Serving on http://:::9283 2024-06-21T01:00:42.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:00:42] ENGINE Bus STARTED 2024-06-21T01:00:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:42.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: Standby manager daemon y started 2024-06-21T01:00:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T01:00:42.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:00:42.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T01:00:42.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: Standby manager daemon y started 2024-06-21T01:00:42.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T01:00:42.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:00:42.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T01:00:42.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:00:42.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:42.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:42 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:43.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:43.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:43.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: Standby manager daemon y started 2024-06-21T01:00:43.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-06-21T01:00:43.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:00:43.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-06-21T01:00:43.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:00:43.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:43.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:42 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:43 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: mgrmap e35: x(active, since 6s), standbys: y 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:43 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:00:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:00:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:44 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:45.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:45.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:00:45.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:44 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:46.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.326 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:00:47.327 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:45 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:00:47.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:46 smithi120 ceph-mon[25975]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:00:47.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:45 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:00:47.330 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:46 smithi027 ceph-mon[25136]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:00:47.331 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:46 smithi027 ceph-mon[29814]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:00:48.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:00:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:00:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T01:00:49.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:48 smithi027 ceph-mon[25136]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:00:49.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:48 smithi027 ceph-mon[29814]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:00:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:48 smithi120 ceph-mon[25975]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:00:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:00:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:00:50.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:00:50.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/728502312' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:00:50.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3715890848' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:50.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:50 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:50.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:00:50.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/728502312' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3715890848' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:00:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/728502312' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3715890848' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:50 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]: dispatch 2024-06-21T01:00:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:51 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]': finished 2024-06-21T01:00:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:51 smithi120 ceph-mon[25975]: osdmap e106: 8 total, 8 up, 8 in 2024-06-21T01:00:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:51 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2967280213' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:51 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]': finished 2024-06-21T01:00:52.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[25136]: osdmap e106: 8 total, 8 up, 8 in 2024-06-21T01:00:52.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2967280213' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2699524388"}]': finished 2024-06-21T01:00:52.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[29814]: osdmap e106: 8 total, 8 up, 8 in 2024-06-21T01:00:52.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2967280213' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:51 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]: dispatch 2024-06-21T01:00:52.447 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:00:52.447 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 bash[57314]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.449Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T01:00:52.754 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[54436]: ts=2024-06-21T01:00:52.450Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 podman[57325]: 2024-06-21 01:00:52.459668936 +0000 UTC m=+0.133593648 container died 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 podman[57325]: 2024-06-21 01:00:52.756639287 +0000 UTC m=+0.430563999 container cleanup 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:52.755 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 bash[57325]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T01:00:53.005 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:52 smithi120 bash[57354]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T01:00:53.005 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-21T01:00:53.005 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]': finished 2024-06-21T01:00:53.005 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: osdmap e107: 8 total, 8 up, 8 in 2024-06-21T01:00:53.005 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4288392093' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.006 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.006 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:52 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]': finished 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: osdmap e107: 8 total, 8 up, 8 in 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4288392093' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/1625480472"}]': finished 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: osdmap e107: 8 total, 8 up, 8 in 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4288392093' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]: dispatch 2024-06-21T01:00:53.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:52 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 podman[57337]: 2024-06-21 01:00:53.008015943 +0000 UTC m=+0.549929233 container remove 9cf45c6f9e840964b76e1a101538893026ffa38db59eba5e1fcd98da95859026 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.389s CPU time. 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 podman[57416]: 2024-06-21 01:00:53.226149939 +0000 UTC m=+0.107527361 container create c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:53.327 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 podman[57416]: 2024-06-21 01:00:53.136705516 +0000 UTC m=+0.018082924 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-21T01:00:53.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 podman[57416]: 2024-06-21 01:00:53.444885773 +0000 UTC m=+0.326263158 container init c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:53.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 podman[57416]: 2024-06-21 01:00:53.447730908 +0000 UTC m=+0.329108298 container start c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:00:53.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.470Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.472Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.472Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.473Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.476Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.476Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.458µs 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.476Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T01:00:53.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 bash[57416]: c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 2024-06-21T01:00:53.618 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:00:54.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.665Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-06-21T01:00:54.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:53.902Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-06-21T01:00:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]': finished 2024-06-21T01:00:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: osdmap e108: 8 total, 8 up, 8 in 2024-06-21T01:00:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2138318571' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:54.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]': finished 2024-06-21T01:00:54.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: osdmap e108: 8 total, 8 up, 8 in 2024-06-21T01:00:54.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:54.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2138318571' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:54.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/18481122"}]': finished 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: osdmap e108: 8 total, 8 up, 8 in 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2138318571' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:00:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:00:54.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.138Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-06-21T01:00:54.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.187Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-06-21T01:00:54.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.187Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-06-21T01:00:54.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.187Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.51µs wal_replay_duration=711.447417ms total_replay_duration=711.497968ms 2024-06-21T01:00:54.395 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.192Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T01:00:54.396 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.192Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T01:00:54.396 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.192Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T01:00:54.396 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.211Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.524802ms db_storage=758ns remote_storage=1.274µs web_handler=312ns query_engine=799ns scrape=371.886µs scrape_sd=122.298µs notify=17.536µs notify_sd=15.542µs rules=18.594748ms 2024-06-21T01:00:54.396 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:00:54.211Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T01:00:54.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]': finished 2024-06-21T01:00:54.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-mon[25975]: osdmap e109: 8 total, 8 up, 8 in 2024-06-21T01:00:54.972 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1640975175' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:54.973 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:54 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]': finished 2024-06-21T01:00:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[25136]: osdmap e109: 8 total, 8 up, 8 in 2024-06-21T01:00:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1640975175' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2142397980"}]': finished 2024-06-21T01:00:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[29814]: osdmap e109: 8 total, 8 up, 8 in 2024-06-21T01:00:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1640975175' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:55.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:54 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]: dispatch 2024-06-21T01:00:55.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:55 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]': finished 2024-06-21T01:00:55.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:55 smithi120 ceph-mon[25975]: osdmap e110: 8 total, 8 up, 8 in 2024-06-21T01:00:55.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:55 smithi120 ceph-mon[25975]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T01:00:55.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:55 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4149992674' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:55.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:55 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:56.095 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]': finished 2024-06-21T01:00:56.095 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[29814]: osdmap e110: 8 total, 8 up, 8 in 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[29814]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4149992674' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2796351247"}]': finished 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[25136]: osdmap e110: 8 total, 8 up, 8 in 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[25136]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4149992674' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:56.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:55 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]: dispatch 2024-06-21T01:00:57.228 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:56 smithi120 ceph-mon[25975]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]': finished 2024-06-21T01:00:57.228 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:56 smithi120 ceph-mon[25975]: osdmap e111: 8 total, 8 up, 8 in 2024-06-21T01:00:57.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:56 smithi027 ceph-mon[25136]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]': finished 2024-06-21T01:00:57.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:56 smithi027 ceph-mon[25136]: osdmap e111: 8 total, 8 up, 8 in 2024-06-21T01:00:57.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:56 smithi027 ceph-mon[29814]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/1625480472"}]': finished 2024-06-21T01:00:57.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:56 smithi027 ceph-mon[29814]: osdmap e111: 8 total, 8 up, 8 in 2024-06-21T01:00:58.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 1 op/s 2024-06-21T01:00:58.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:58.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:00:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 1 op/s 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 1 op/s 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:00:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:00:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:00:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:00:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:01:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:00 smithi120 ceph-mon[25975]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T01:01:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:00 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:00.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:00 smithi027 ceph-mon[25136]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T01:01:00.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:00 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:00.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:00 smithi027 ceph-mon[29814]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-06-21T01:01:00.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:00 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:02 smithi120 ceph-mon[25975]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-06-21T01:01:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:02 smithi027 ceph-mon[25136]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-06-21T01:01:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:02 smithi027 ceph-mon[29814]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 724 B/s rd, 0 op/s 2024-06-21T01:01:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:04 smithi120 ceph-mon[25975]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-06-21T01:01:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:04 smithi027 ceph-mon[25136]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-06-21T01:01:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:04 smithi027 ceph-mon[29814]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-06-21T01:01:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:06 smithi120 ceph-mon[25975]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:01:07.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:06 smithi027 ceph-mon[25136]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:01:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:06 smithi027 ceph-mon[29814]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:01:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:07 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:07 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:07 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:08] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T01:01:09.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:08 smithi120 ceph-mon[25975]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-06-21T01:01:09.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:08 smithi027 ceph-mon[25136]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-06-21T01:01:09.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:08 smithi027 ceph-mon[29814]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 926 B/s rd, 0 op/s 2024-06-21T01:01:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:10 smithi120 ceph-mon[25975]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:10 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:10 smithi027 ceph-mon[25136]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:10 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:10 smithi027 ceph-mon[29814]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:10 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:12 smithi120 ceph-mon[25975]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:12 smithi027 ceph-mon[25136]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:12 smithi027 ceph-mon[29814]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:14 smithi120 ceph-mon[25975]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:14 smithi027 ceph-mon[25136]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:15.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:14 smithi027 ceph-mon[29814]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:16 smithi120 ceph-mon[25975]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:16 smithi027 ceph-mon[25136]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:16 smithi027 ceph-mon[29814]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:18] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T01:01:19.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:18 smithi120 ceph-mon[25975]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:19.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:18 smithi027 ceph-mon[25136]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:19.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:18 smithi027 ceph-mon[29814]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:20 smithi120 ceph-mon[25975]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:20 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:21.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:20 smithi027 ceph-mon[29814]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:21.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:20 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:20 smithi027 ceph-mon[25136]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:20 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:22 smithi120 ceph-mon[25975]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:22 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:22 smithi027 ceph-mon[25136]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:23.317 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:22 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:22 smithi027 ceph-mon[29814]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:22 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:24 smithi120 ceph-mon[25975]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:25.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:24 smithi027 ceph-mon[25136]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:24 smithi027 ceph-mon[29814]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:26 smithi120 ceph-mon[25975]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:27.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:26 smithi027 ceph-mon[25136]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:26 smithi027 ceph-mon[29814]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:28] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-06-21T01:01:29.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:28 smithi120 ceph-mon[25975]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:28 smithi027 ceph-mon[25136]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:28 smithi027 ceph-mon[29814]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:31.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:30 smithi120 ceph-mon[25975]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:31.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:30 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:30 smithi027 ceph-mon[29814]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:30 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:30 smithi027 ceph-mon[25136]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:30 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:32 smithi120 ceph-mon[25975]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:32 smithi027 ceph-mon[25136]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:32 smithi027 ceph-mon[29814]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:34 smithi027 ceph-mon[25136]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:34 smithi027 ceph-mon[29814]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:35.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:34 smithi120 ceph-mon[25975]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:36 smithi027 ceph-mon[25136]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:36 smithi027 ceph-mon[29814]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:36 smithi120 ceph-mon[25975]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:38.344 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:38] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T01:01:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:38 smithi027 ceph-mon[25136]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:39.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:38 smithi027 ceph-mon[29814]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:39.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:38 smithi120 ceph-mon[25975]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:39 smithi027 ceph-mon[25136]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:39 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:40.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:39 smithi027 ceph-mon[29814]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:39 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:39 smithi120 ceph-mon[25975]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:39 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:42 smithi120 ceph-mon[25975]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:42 smithi027 ceph-mon[25136]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:43.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:42 smithi027 ceph-mon[29814]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:44 smithi120 ceph-mon[25975]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:45.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:44 smithi027 ceph-mon[25136]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:45.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:44 smithi027 ceph-mon[29814]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:46 smithi120 ceph-mon[25975]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:46 smithi027 ceph-mon[25136]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:47.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:46 smithi027 ceph-mon[29814]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:48] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T01:01:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:48 smithi120 ceph-mon[25975]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:49.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:48 smithi027 ceph-mon[25136]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:49.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:48 smithi027 ceph-mon[29814]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:51.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:50 smithi120 ceph-mon[25975]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:51.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:50 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:51.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:50 smithi027 ceph-mon[25136]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:51.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:50 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:51.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:50 smithi027 ceph-mon[29814]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:51.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:50 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:01:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:52 smithi120 ceph-mon[25975]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:52 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:52 smithi027 ceph-mon[25136]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:52 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:52 smithi027 ceph-mon[29814]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:52 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:01:55.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:54 smithi120 ceph-mon[25975]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:54 smithi027 ceph-mon[25136]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:55.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:54 smithi027 ceph-mon[29814]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:57.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:56 smithi120 ceph-mon[25975]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:56 smithi027 ceph-mon[25136]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:56 smithi027 ceph-mon[29814]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:01:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:01:58] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-06-21T01:01:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-mon[25975]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:01:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:01:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:01:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:01:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:01:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[25136]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:01:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:01:59.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[29814]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:01:59.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:01:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:02:01.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:00 smithi120 ceph-mon[25975]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:01.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:00 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:00 smithi027 ceph-mon[29814]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:00 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:00 smithi027 ceph-mon[25136]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:00 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:02 smithi120 ceph-mon[25975]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:02 smithi027 ceph-mon[25136]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:02 smithi027 ceph-mon[29814]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:04 smithi120 ceph-mon[25975]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:04 smithi027 ceph-mon[25136]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:04 smithi027 ceph-mon[29814]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:06 smithi120 ceph-mon[25975]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:06 smithi027 ceph-mon[25136]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:07.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:06 smithi027 ceph-mon[29814]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:07 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:07 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:07 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:08] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-06-21T01:02:09.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:08 smithi120 ceph-mon[25975]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:09.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:08 smithi027 ceph-mon[25136]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:09.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:08 smithi027 ceph-mon[29814]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:10 smithi120 ceph-mon[25975]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:10 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:10 smithi027 ceph-mon[29814]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:10 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:10 smithi027 ceph-mon[25136]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:11.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:10 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:12 smithi027 ceph-mon[25136]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:12 smithi027 ceph-mon[29814]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:13.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:12 smithi120 ceph-mon[25975]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:14 smithi027 ceph-mon[25136]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:14 smithi027 ceph-mon[29814]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:14 smithi120 ceph-mon[25975]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:16 smithi027 ceph-mon[25136]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:16 smithi027 ceph-mon[29814]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:16 smithi120 ceph-mon[25975]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:18] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-06-21T01:02:19.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:18 smithi027 ceph-mon[25136]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:19.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:18 smithi027 ceph-mon[29814]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:19.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:18 smithi120 ceph-mon[25975]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:20.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:19 smithi027 ceph-mon[25136]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:20.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:19 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:20.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:19 smithi027 ceph-mon[29814]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:20.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:19 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:20.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:19 smithi120 ceph-mon[25975]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:20.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:19 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:22 smithi120 ceph-mon[25975]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:22 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:22 smithi027 ceph-mon[25136]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:22 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:22 smithi027 ceph-mon[29814]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:22 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:24 smithi120 ceph-mon[25975]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:25.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:24 smithi027 ceph-mon[25136]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:25.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:24 smithi027 ceph-mon[29814]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:26 smithi120 ceph-mon[25975]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:26 smithi027 ceph-mon[25136]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:27.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:26 smithi027 ceph-mon[29814]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:28] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-06-21T01:02:29.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:28 smithi120 ceph-mon[25975]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:28 smithi027 ceph-mon[25136]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:28 smithi027 ceph-mon[29814]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:31.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:30 smithi120 ceph-mon[25975]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:31.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:30 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:30 smithi027 ceph-mon[25136]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:30 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:30 smithi027 ceph-mon[29814]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:30 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:32 smithi120 ceph-mon[25975]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:32 smithi027 ceph-mon[25136]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:32 smithi027 ceph-mon[29814]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:34 smithi120 ceph-mon[25975]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:34 smithi027 ceph-mon[25136]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:34 smithi027 ceph-mon[29814]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:36 smithi120 ceph-mon[25975]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:36 smithi027 ceph-mon[25136]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:36 smithi027 ceph-mon[29814]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:38] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-06-21T01:02:39.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:38 smithi120 ceph-mon[25975]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:38 smithi027 ceph-mon[25136]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:39.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:38 smithi027 ceph-mon[29814]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:41.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:40 smithi120 ceph-mon[25975]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:41.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:40 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:41.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:40 smithi027 ceph-mon[25136]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:41.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:40 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:41.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:40 smithi027 ceph-mon[29814]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:41.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:40 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:42 smithi120 ceph-mon[25975]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:42 smithi027 ceph-mon[25136]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:42 smithi027 ceph-mon[29814]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:44 smithi027 ceph-mon[25136]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:44 smithi027 ceph-mon[29814]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:45.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:44 smithi120 ceph-mon[25975]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:47.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:46 smithi027 ceph-mon[25136]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:47.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:46 smithi027 ceph-mon[29814]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:46 smithi120 ceph-mon[25975]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:48] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-06-21T01:02:49.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:48 smithi027 ceph-mon[25136]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:49.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:48 smithi027 ceph-mon[29814]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:49.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:48 smithi120 ceph-mon[25975]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:51.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:50 smithi027 ceph-mon[25136]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:51.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:50 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:51.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:50 smithi027 ceph-mon[29814]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:51.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:50 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:50 smithi120 ceph-mon[25975]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:50 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:02:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:51 smithi027 ceph-mon[25136]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:51 smithi027 ceph-mon[29814]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:51 smithi120 ceph-mon[25975]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:52 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:53.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:52 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:53.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:52 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:02:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:53 smithi027 ceph-mon[25136]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:53 smithi027 ceph-mon[29814]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:54.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:53 smithi120 ceph-mon[25975]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:56 smithi120 ceph-mon[25975]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:56 smithi027 ceph-mon[25136]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:56 smithi027 ceph-mon[29814]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:02:58.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:02:58] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-21T01:02:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-mon[25975]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:59.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:02:59.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:02:59.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:02:59.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:02:58 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:02:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[25136]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:02:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:02:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:02:59.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:02:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[29814]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:02:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:02:59.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:02:59.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:02:59.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:02:58 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:01.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:00 smithi120 ceph-mon[25975]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:01.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:00 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:00 smithi027 ceph-mon[25136]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:00 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:00 smithi027 ceph-mon[29814]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:01.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:00 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:02 smithi120 ceph-mon[25975]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:02 smithi027 ceph-mon[25136]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:02 smithi027 ceph-mon[29814]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:04 smithi120 ceph-mon[25975]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:04 smithi027 ceph-mon[25136]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:04 smithi027 ceph-mon[29814]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:06 smithi120 ceph-mon[25975]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:06 smithi027 ceph-mon[25136]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:07.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:06 smithi027 ceph-mon[29814]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:07 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:07 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:07 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:03:08] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:03:09.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:08 smithi120 ceph-mon[25975]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:09.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:08 smithi027 ceph-mon[25136]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:09.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:08 smithi027 ceph-mon[29814]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:10 smithi120 ceph-mon[25975]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:10 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:10 smithi027 ceph-mon[25136]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:10 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:10 smithi027 ceph-mon[29814]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:10 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:12 smithi120 ceph-mon[25975]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:12 smithi027 ceph-mon[25136]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:12 smithi027 ceph-mon[29814]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:14 smithi027 ceph-mon[25136]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:15.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:14 smithi027 ceph-mon[29814]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:14 smithi120 ceph-mon[25975]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:16 smithi027 ceph-mon[25136]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:16 smithi027 ceph-mon[29814]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:16 smithi120 ceph-mon[25975]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:18.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:03:18] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:03:19.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:18 smithi027 ceph-mon[25136]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:19.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:18 smithi027 ceph-mon[29814]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:19.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:18 smithi120 ceph-mon[25975]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:20 smithi027 ceph-mon[25136]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:20 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:20 smithi027 ceph-mon[29814]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:20 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:20 smithi120 ceph-mon[25975]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:20 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:22.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:21 smithi027 ceph-mon[25136]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:21 smithi027 ceph-mon[29814]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:21 smithi120 ceph-mon[25975]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:22 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:22 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:22 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:23 smithi027 ceph-mon[25136]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:23 smithi027 ceph-mon[29814]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:24.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:23 smithi120 ceph-mon[25975]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:26 smithi120 ceph-mon[25975]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:26 smithi027 ceph-mon[25136]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:26 smithi027 ceph-mon[29814]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:28.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:03:28] "GET /metrics HTTP/1.1" 200 37915 "" "Prometheus/2.33.4" 2024-06-21T01:03:29.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:28 smithi120 ceph-mon[25975]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:28 smithi027 ceph-mon[25136]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:28 smithi027 ceph-mon[29814]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:31.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:30 smithi120 ceph-mon[25975]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:32.046 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:30 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:32.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:30 smithi027 ceph-mon[25136]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:32.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:30 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:32.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:30 smithi027 ceph-mon[29814]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:32.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:30 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:32 smithi120 ceph-mon[25975]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:32 smithi027 ceph-mon[25136]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:32 smithi027 ceph-mon[29814]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:34 smithi120 ceph-mon[25975]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:34 smithi027 ceph-mon[25136]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:34 smithi027 ceph-mon[29814]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:36 smithi120 ceph-mon[25975]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:36 smithi027 ceph-mon[25136]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:36 smithi027 ceph-mon[29814]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:37 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:37 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:38.344 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:37 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:38.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:03:38] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:03:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:38 smithi027 ceph-mon[25136]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:39.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:38 smithi027 ceph-mon[29814]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:39.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:38 smithi120 ceph-mon[25975]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:40.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:03:41.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:40 smithi027 ceph-mon[29814]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:41.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:40 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:41.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:40 smithi027 ceph-mon[25136]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:41.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:40 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:40 smithi120 ceph-mon[25975]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:40 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (12m) 2m ago 20m 22.9M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (12m) 2m ago 20m 55.0M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 2m ago 19m 74.9M - 3.5 e1d6a67b021e ab94feaf9653 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (15m) 2m ago 25m 489M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (9m) 2m ago 26m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (26m) 2m ago 26m 67.6M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (25m) 2m ago 25m 54.0M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (25m) 2m ago 25m 55.7M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:03:42.280 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 2m ago 21m 16.5M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (12m) 2m ago 21m 16.8M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (24m) 2m ago 24m 53.8M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (24m) 2m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (23m) 2m ago 23m 48.9M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (23m) 2m ago 23m 53.8M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (23m) 2m ago 23m 53.9M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (22m) 2m ago 22m 52.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (22m) 2m ago 22m 50.7M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (22m) 2m ago 22m 53.7M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 2m ago 21m 37.7M - 2.33.4 514e6a882f6e c48c03760486 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (19m) 2m ago 19m 78.0M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:03:42.281 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (19m) 2m ago 19m 76.0M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:03:42.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ls' 2024-06-21T01:03:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:42 smithi027 ceph-mon[25136]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:42 smithi027 ceph-mon[29814]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:43.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:42 smithi120 ceph-mon[25975]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:44.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:43 smithi027 ceph-mon[29814]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:44.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:43 smithi027 ceph-mon[25136]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:44.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:43 smithi120 ceph-mon[25975]: from='client.25366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi027=a;count:1 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:grafana ?:3000 1/1 2m ago 21m smithi120=a;count:1 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:mgr 2/2 2m ago 25m smithi027=y;smithi120=x;count:2 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:mon 3/3 2m ago 25m smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi120:172.21.15.120=b;count:3 2024-06-21T01:03:44.902 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi027=a;smithi120=b;count:2 2024-06-21T01:03:44.903 INFO:teuthology.orchestra.run.smithi027.stdout:osd 8 2m ago - 2024-06-21T01:03:44.903 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus ?:9095 1/1 2m ago 21m smithi120=a;count:1 2024-06-21T01:03:44.903 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-06-21T01:03:45.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:44 smithi027 ceph-mon[29814]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:44 smithi027 ceph-mon[25136]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:45.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:44 smithi120 ceph-mon[25975]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:45.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T01:03:47.090 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:46 smithi027 ceph-mon[25136]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:47.090 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:46 smithi027 ceph-mon[25136]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:47.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:46 smithi027 ceph-mon[29814]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:47.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:46 smithi027 ceph-mon[29814]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:46 smithi120 ceph-mon[25975]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:46 smithi120 ceph-mon[25975]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:47.624 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:03:47.624 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:03:47.625 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:03:47.626 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:03:47.626 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:03:47.626 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:03:48.151 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:47 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3516651538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:48.151 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:47 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3516651538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:48.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-06-21T01:03:48.344 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:47 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3516651538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:48.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:03:48] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:03:49.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:48 smithi027 ceph-mon[29814]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:49.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:48 smithi027 ceph-mon[25136]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:49.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:48 smithi120 ceph-mon[25975]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:51.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:50 smithi027 ceph-mon[25136]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:51.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:50 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:51.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:50 smithi027 ceph-mon[29814]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:51.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:50 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:50 smithi120 ceph-mon[25975]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:50 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:03:52.126 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:52.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:51 smithi027 ceph-mon[25136]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:52.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:51 smithi027 ceph-mon[25136]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:52.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:51 smithi027 ceph-mon[29814]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:52.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:51 smithi027 ceph-mon[29814]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:51 smithi120 ceph-mon[25975]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:51 smithi120 ceph-mon[25975]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:52.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:03:53.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:53 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:03:53.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:53.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:03:53.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:53 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:54 smithi027 ceph-mon[25136]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:54 smithi027 ceph-mon[29814]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:54.517 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:54 smithi120 ceph-mon[25975]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:03:55.956 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.278+0000 7f7a79ae1640 -1 mgr handle_mgr_map I was active but no longer am 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ignoring --setuser ceph since I am not root 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ignoring --setgroup ceph since I am not root 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mgr[46008]: -- 172.21.15.120:0/487281225 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c5d12d30e0 con 0x55c5d12b1000 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.430+0000 7fd035c5e040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T01:03:56.578 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.579+0000 7fd035c5e040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Failing over to other MGR 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.579 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:03:56.580 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T01:03:56.689 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE Bus STOPPING 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.690 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Failing over to other MGR 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Upgrade: Need to upgrade myself (mgr.x) 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Failing over to other MGR 2024-06-21T01:03:56.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: from='mgr.35024 172.21.15.120:0/3726315225' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-06-21T01:03:56.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:03:56.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T01:03:56.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.649+0000 7fd035c5e040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T01:03:56.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE Bus STOPPED 2024-06-21T01:03:56.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE Bus STARTING 2024-06-21T01:03:56.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE Serving on http://:::9283 2024-06-21T01:03:56.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:56 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:03:56] ENGINE Bus STARTED 2024-06-21T01:03:57.245 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.903+0000 7fd035c5e040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T01:03:57.245 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:56 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:56.963+0000 7fd035c5e040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: mgrmap e36: y(active, starting, since 1.00541s), standbys: x 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:03:57.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: Manager daemon y is now available 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:03:57.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: mgrmap e36: y(active, starting, since 1.00541s), standbys: x 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:03:57.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: Manager daemon y is now available 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:03:57.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:57 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:03:57.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.35024 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-06-21T01:03:57.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: mgrmap e36: y(active, starting, since 1.00541s), standbys: x 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:03:57.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: Manager daemon y is now available 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:03:57.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:57 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (12m) 2m ago 20m 22.9M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (12m) 3m ago 20m 55.0M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (3m) 2m ago 19m 74.9M - 3.5 e1d6a67b021e ab94feaf9653 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283 running (15m) 3m ago 25m 489M - 19.0.0-2541-gc65be810 e9bfc9e474e5 870c0b47af55 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (9m) 2m ago 27m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (27m) 2m ago 27m 67.6M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:03:57.770 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (25m) 3m ago 25m 54.0M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (25m) 2m ago 25m 55.7M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 2m ago 21m 16.5M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (12m) 3m ago 21m 16.8M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (25m) 2m ago 25m 53.8M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (24m) 2m ago 24m 56.9M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (24m) 2m ago 24m 48.9M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (23m) 2m ago 23m 53.8M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (23m) 3m ago 23m 53.9M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (23m) 3m ago 23m 52.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (22m) 3m ago 22m 50.7M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (22m) 3m ago 22m 53.7M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (3m) 3m ago 21m 37.7M - 2.33.4 514e6a882f6e c48c03760486 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (20m) 2m ago 20m 78.0M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:03:57.771 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (19m) 3m ago 19m 76.0M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:03:58.214 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:03:58.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:03:58.216 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:03:58.216 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:03:58.216 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:03:58.216 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:03:58.216 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:03:58.287 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.006+0000 7fd035c5e040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T01:03:58.287 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.131+0000 7fd035c5e040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T01:03:58.288 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.190+0000 7fd035c5e040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T01:03:58.445 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[25136]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-21T01:03:58.445 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[25136]: from='client.25396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.445 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[25136]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:03:58.445 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[25136]: from='client.35225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3913739523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[29814]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[29814]: from='client.25396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[29814]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[29814]: from='client.35225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:58 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3913739523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:58.553 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.414+0000 7fd035c5e040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.483+0000 7fd035c5e040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-mon[25975]: mgrmap e37: y(active, since 2s), standbys: x 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-mon[25975]: from='client.25396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-mon[25975]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-mon[25975]: from='client.35225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:03:58.554 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3913739523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:03:58.572 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:03:58.572 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:03:58.572 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:03:58.572 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-21T01:03:58.572 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:03:58.573 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:03:58.811 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.555+0000 7fd035c5e040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T01:03:58.811 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:58 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:58.708+0000 7fd035c5e040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:03:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:03:59.014+0000 7fbdd9c18640 -1 log_channel(cephadm) log [ERR] : [21/Jun/2024:01:03:59] ENGINE Error in HTTPServer.serve 2024-06-21T01:03:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T01:03:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._connections.run(self.expiration_interval) 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._run(expiration_interval) 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s = self.context.wrap_socket( 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:03:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.sslsocket_class._create( 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self.do_handshake() 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._sslobj.do_handshake() 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:03:59.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:03:59.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[25136]: [21/Jun/2024:01:03:58] ENGINE Bus STARTING 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[25136]: [21/Jun/2024:01:03:58] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[25136]: [21/Jun/2024:01:03:59] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[25136]: [21/Jun/2024:01:03:59] ENGINE Bus STARTED 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[25136]: [21/Jun/2024:01:03:59] ENGINE Error in HTTPServer.serve 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:03:59.451 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:03:59.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[29814]: [21/Jun/2024:01:03:58] ENGINE Bus STARTING 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[29814]: [21/Jun/2024:01:03:58] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[29814]: [21/Jun/2024:01:03:59] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[29814]: [21/Jun/2024:01:03:59] ENGINE Bus STARTED 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:03:59 smithi027 ceph-mon[29814]: [21/Jun/2024:01:03:59] ENGINE Error in HTTPServer.serve 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:03:59.453 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:03:59.454 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:59.152+0000 7fd035c5e040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-mon[25975]: [21/Jun/2024:01:03:58] ENGINE Bus STARTING 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-mon[25975]: [21/Jun/2024:01:03:58] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-mon[25975]: [21/Jun/2024:01:03:59] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-mon[25975]: [21/Jun/2024:01:03:59] ENGINE Bus STARTED 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-mon[25975]: [21/Jun/2024:01:03:59] ENGINE Error in HTTPServer.serve 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:03:59.514 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._run(expiration_interval) 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: s = self.context.wrap_socket( 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.sslsocket_class._create( 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.do_handshake() 2024-06-21T01:03:59.515 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:03:59.516 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._sslobj.do_handshake() 2024-06-21T01:03:59.516 INFO:journalctl@ceph.mon.b.smithi120.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:03:59.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:59.516+0000 7fd035c5e040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T01:03:59.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:59.584+0000 7fd035c5e040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T01:03:59.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:03:59 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:03:59.757+0000 7fd035c5e040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T01:04:00.385 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-mon[25975]: mgrmap e38: y(active, since 4s), standbys: x 2024-06-21T01:04:00.385 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-mon[25975]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:00.385 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.126+0000 7fd035c5e040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T01:04:00.386 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.190+0000 7fd035c5e040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T01:04:00.386 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.387+0000 7fd035c5e040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T01:04:00.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:00 smithi027 ceph-mon[25136]: mgrmap e38: y(active, since 4s), standbys: x 2024-06-21T01:04:00.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:00 smithi027 ceph-mon[25136]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:00.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:00 smithi027 ceph-mon[29814]: mgrmap e38: y(active, since 4s), standbys: x 2024-06-21T01:04:00.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:00 smithi027 ceph-mon[29814]: from='client.25315 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:00.690 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.540+0000 7fd035c5e040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T01:04:00.690 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.607+0000 7fd035c5e040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T01:04:00.691 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.692+0000 7fd035c5e040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T01:04:00.987 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:00.757+0000 7fd035c5e040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T01:04:01.607 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:01.328+0000 7fd035c5e040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T01:04:01.607 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:01.502+0000 7fd035c5e040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:04:01] ENGINE Bus STARTING 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: CherryPy Checker: 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: The Application mounted at '' has an empty config. 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:04:01] ENGINE Serving on http://:::9283 2024-06-21T01:04:01.608 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: [21/Jun/2024:01:04:01] ENGINE Bus STARTED 2024-06-21T01:04:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: mgrmap e39: y(active, since 6s), standbys: x 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: Standby manager daemon x restarted 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:01.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:01.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:01.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:01 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: mgrmap e39: y(active, since 6s), standbys: x 2024-06-21T01:04:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: Standby manager daemon x restarted 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: mgrmap e39: y(active, since 6s), standbys: x 2024-06-21T01:04:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: Standby manager daemon x restarted 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:02.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:01 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/938967844' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:03.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:02 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: mgrmap e40: y(active, since 7s), standbys: x 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:02 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:04.203 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:04.203 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:04:04.203 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:04.204 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:04.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:03 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:04:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:04.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:03 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[25136]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[25136]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:05.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:04 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:04 smithi120 ceph-mon[25975]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:04 smithi120 ceph-mon[25975]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:04:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:04 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:04 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:05 smithi027 ceph-mon[25136]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s rd, 0 B/s wr, 207 op/s 2024-06-21T01:04:05.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:05 smithi027 ceph-mon[29814]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s rd, 0 B/s wr, 207 op/s 2024-06-21T01:04:06.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:05 smithi120 ceph-mon[25975]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s rd, 0 B/s wr, 207 op/s 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:04:06.506+0000 7fbe16f4f640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Creating ceph-iscsi config... 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:06.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return _run_code(code, main_globals, None, 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: exec(code, run_globals) 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:06.730 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:06.731 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.wait_async( 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.event_loop.get_result(coro, timeout) 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return future.result(timeout) 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T01:04:06.732 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.__get_result() 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: raise self._exception 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: out, err, code = await self._run_cephadm( 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: raise OrchestratorError( 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Creating ceph-iscsi config... 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:06.733 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return _run_code(code, main_globals, None, 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: exec(code, run_globals) 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:06.734 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:06.735 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:06.735 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:06.735 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:06.735 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:06 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[29814]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 0 B/s wr, 161 op/s 2024-06-21T01:04:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[29814]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.wait_async( 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T01:04:08.230 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[25136]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 0 B/s wr, 161 op/s 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[25136]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.231 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T01:04:08.232 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.wait_async( 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: return future.result(timeout) 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.__get_result() 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise self._exception 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: raise OrchestratorError( 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.233 INFO:journalctl@ceph.mon.a.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: exec(code, run_globals) 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.234 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[25136]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[25136]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.c.smithi027.stdout: return future.result(timeout) 2024-06-21T01:04:08.235 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.__get_result() 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise self._exception 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: out, err, code = await self._run_cephadm( 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: raise OrchestratorError( 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.236 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.237 INFO:journalctl@ceph.mon.c.smithi027.stdout: exec(code, run_globals) 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.239 INFO:journalctl@ceph.mon.c.smithi027.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[29814]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:04:08.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:07 smithi027 ceph-mon[29814]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:04:08.333 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:07 smithi120 ceph-mon[25975]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 0 B/s wr, 161 op/s 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:07 smithi120 ceph-mon[25975]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T01:04:08.334 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2456, in _daemon_action 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.wait_async( 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 779, in wait_async 2024-06-21T01:04:08.335 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.event_loop.get_result(coro, timeout) 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: return future.result(timeout) 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.__get_result() 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise self._exception 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1350, in _create_daemon 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: out, err, code = await self._run_cephadm( 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1691, in _run_cephadm 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: raise OrchestratorError( 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: Creating ceph-iscsi config... 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:04:08.336 INFO:journalctl@ceph.mon.b.smithi120.stdout: Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: Non-zero exit code 1 from systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: exec(code, run_globals) 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1123, in deploy_daemon 2024-06-21T01:04:08.337 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:04:08.338 INFO:journalctl@ceph.mon.b.smithi120.stdout: RuntimeError: Failed command: systemctl restart ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:04:08.338 INFO:journalctl@ceph.mon.b.smithi120.stdout: See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:04:08.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:07 smithi120 ceph-mon[25975]: Reconfiguring prometheus.a (dependencies changed)... 2024-06-21T01:04:08.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:07 smithi120 ceph-mon[25975]: Reconfiguring daemon prometheus.a on smithi120 2024-06-21T01:04:08.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[46004]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-06-21T01:04:09.920 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-mon[25975]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 0 B/s wr, 132 op/s 2024-06-21T01:04:09.920 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:09.921 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 bash[60147]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T01:04:10.188 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.923Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T01:04:10.189 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.924Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T01:04:10.189 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.924Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T01:04:10.189 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[57432]: ts=2024-06-21T01:04:09.924Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T01:04:10.189 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:09 smithi120 podman[60159]: 2024-06-21 01:04:09.933345185 +0000 UTC m=+0.108555390 container died c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:09 smithi027 ceph-mon[25136]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 0 B/s wr, 132 op/s 2024-06-21T01:04:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:09 smithi027 ceph-mon[29814]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 0 B/s wr, 132 op/s 2024-06-21T01:04:10.587 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 podman[60159]: 2024-06-21 01:04:10.280689729 +0000 UTC m=+0.455899933 container cleanup c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:10.587 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 bash[60159]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T01:04:10.587 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 bash[60184]: Error: no container with name or ID "ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus.a" found: no such container 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 podman[60171]: 2024-06-21 01:04:10.590031761 +0000 UTC m=+0.657777844 container remove c48c03760486ba7c4a679ce14d7bf2666b0a2b08d6487f4cdcea8994be3eb967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.412s CPU time. 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 podman[60252]: 2024-06-21 01:04:10.731631211 +0000 UTC m=+0.018491447 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-06-21T01:04:10.852 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:10 smithi120 podman[60252]: 2024-06-21 01:04:10.855012302 +0000 UTC m=+0.141872527 container create d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:11.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 podman[60252]: 2024-06-21 01:04:11.118449389 +0000 UTC m=+0.405309627 container init d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 podman[60252]: 2024-06-21 01:04:11.121410054 +0000 UTC m=+0.408270288 container start d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:518 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.144Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.146Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.146Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.148Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.149Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T01:04:11.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.150Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.785µs 2024-06-21T01:04:11.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.150Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T01:04:11.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 bash[60252]: d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 2024-06-21T01:04:11.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:11.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.346Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-06-21T01:04:11.865 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.598Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-06-21T01:04:11.866 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.823Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-06-21T01:04:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.868Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.871Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.872Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.872Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.91µs wal_replay_duration=722.058559ms total_replay_duration=722.114977ms 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.876Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.876Z caller=main.go:947 level=info msg="TSDB started" 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.876Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T01:04:12.117 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.901Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.481031ms db_storage=768ns remote_storage=1.41µs web_handler=491ns query_engine=688ns scrape=371.501µs scrape_sd=121.085µs notify=9.9µs notify_sd=8.92µs rules=23.560236ms 2024-06-21T01:04:12.118 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:04:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:04:11.901Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:12 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:12 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:04:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:12 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:04:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:12 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:04:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:04:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:04:13.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:12 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:13.976 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:13 smithi120 ceph-mon[25975]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:13.976 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:13 smithi120 ceph-mon[25975]: Upgrade: Updating mgr.x 2024-06-21T01:04:13.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:13 smithi120 ceph-mon[25975]: Deploying daemon mgr.x on smithi120 2024-06-21T01:04:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[25136]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[25136]: Upgrade: Updating mgr.x 2024-06-21T01:04:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[25136]: Deploying daemon mgr.x on smithi120 2024-06-21T01:04:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[29814]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:14.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[29814]: Upgrade: Updating mgr.x 2024-06-21T01:04:14.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:13 smithi027 ceph-mon[29814]: Deploying daemon mgr.x on smithi120 2024-06-21T01:04:14.573 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:14 smithi120 systemd[1]: Stopping Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:14.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:14 smithi120 podman[60600]: 2024-06-21 01:04:14.575229124 +0000 UTC m=+0.125356471 container died 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240618) 2024-06-21T01:04:15.242 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:14 smithi120 podman[60600]: 2024-06-21 01:04:14.877677336 +0000 UTC m=+0.427804724 container cleanup 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2) 2024-06-21T01:04:15.242 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:14 smithi120 bash[60600]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x 2024-06-21T01:04:15.242 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 podman[60612]: 2024-06-21 01:04:15.120530849 +0000 UTC m=+0.548391187 container remove 870c0b47af5529585fcbd87a27b4a8a8f4668963424096b2745d772c405081df (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:04:15.242 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-06-21T01:04:15.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-06-21T01:04:15.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 systemd[1]: Stopped Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:15.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service: Consumed 31.726s CPU time. 2024-06-21T01:04:15.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 systemd[1]: Starting Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:15.946 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:15 smithi120 ceph-mon[25975]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:15.947 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 podman[60740]: 2024-06-21 01:04:15.574096463 +0000 UTC m=+0.017206168 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:04:15.947 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 podman[60740]: 2024-06-21 01:04:15.705405099 +0000 UTC m=+0.148514790 container create 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81) 2024-06-21T01:04:15.947 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 podman[60740]: 2024-06-21 01:04:15.949567303 +0000 UTC m=+0.392677003 container init 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-06-21T01:04:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:15 smithi027 ceph-mon[25136]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:15 smithi027 ceph-mon[29814]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 121 op/s 2024-06-21T01:04:16.238 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 podman[60740]: 2024-06-21 01:04:15.952566385 +0000 UTC m=+0.395676078 container start 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=squid-release-c65be81, ceph=True, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:04:16.239 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:15 smithi120 ceph-mgr[60759]: -- 172.21.15.120:0/2859613036 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56176819b0e0 con 0x561768179400 2024-06-21T01:04:16.239 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 bash[60740]: 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 2024-06-21T01:04:16.239 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 systemd[1]: Started Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:16.239 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:16.078+0000 7fcce7c9e040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T01:04:16.239 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:16.240+0000 7fcce7c9e040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:04:16.584 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:16.313+0000 7fcce7c9e040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T01:04:16.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:16.585+0000 7fcce7c9e040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T01:04:16.866 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:16.647+0000 7fcce7c9e040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T01:04:17.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[25136]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:17.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:17 smithi027 ceph-mon[29814]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:17.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-mon[25975]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:17.774 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:17.655+0000 7fcce7c9e040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T01:04:18.094 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:17.776+0000 7fcce7c9e040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T01:04:18.094 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:17.836+0000 7fcce7c9e040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T01:04:18.094 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:18.095+0000 7fcce7c9e040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T01:04:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/956048833' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:18.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:18.161+0000 7fcce7c9e040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T01:04:18.367 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:18.231+0000 7fcce7c9e040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T01:04:18.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:18 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/956048833' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:18.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:18 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/956048833' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:18.849 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:18.389+0000 7fcce7c9e040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:04:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:18] "GET /metrics HTTP/1.1" 200 37916 "" "Prometheus/2.33.4" 2024-06-21T01:04:19.103 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:18.851+0000 7fcce7c9e040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T01:04:19.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-mon[25975]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:19.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:19.197+0000 7fcce7c9e040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T01:04:19.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:19.268+0000 7fcce7c9e040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T01:04:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:19 smithi027 ceph-mon[25136]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:19.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:19 smithi027 ceph-mon[29814]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-21T01:04:19.756 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:19.425+0000 7fcce7c9e040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T01:04:20.019 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:19.757+0000 7fcce7c9e040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T01:04:20.019 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:19.821+0000 7fcce7c9e040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T01:04:20.330 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.021+0000 7fcce7c9e040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T01:04:20.330 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.176+0000 7fcce7c9e040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T01:04:20.331 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.242+0000 7fcce7c9e040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T01:04:20.331 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.332+0000 7fcce7c9e040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T01:04:20.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.399+0000 7fcce7c9e040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T01:04:21.262 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:20.981+0000 7fcce7c9e040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:21.157+0000 7fcce7c9e040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:04:21] ENGINE Bus STARTING 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: CherryPy Checker: 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: The Application mounted at '' has an empty config. 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:04:21] ENGINE Serving on http://:::9283 2024-06-21T01:04:21.263 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:04:21] ENGINE Bus STARTED 2024-06-21T01:04:21.535 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:21.535 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.535 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: Standby manager daemon x restarted 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: Standby manager daemon x started 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:21.536 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:21 smithi120 ceph-mon[25975]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: Standby manager daemon x restarted 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: Standby manager daemon x started 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[25136]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: Standby manager daemon x restarted 2024-06-21T01:04:21.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T01:04:21.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:04:21.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:04:21.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:04:21.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:21 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/2904757861' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:04:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:23 smithi120 ceph-mon[25975]: mgrmap e41: y(active, since 26s), standbys: x 2024-06-21T01:04:23.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:23 smithi027 ceph-mon[25136]: mgrmap e41: y(active, since 26s), standbys: x 2024-06-21T01:04:23.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:23 smithi027 ceph-mon[29814]: mgrmap e41: y(active, since 26s), standbys: x 2024-06-21T01:04:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:24 smithi120 ceph-mon[25975]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:24.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:24 smithi027 ceph-mon[25136]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:24.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:24 smithi027 ceph-mon[29814]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:25 smithi120 ceph-mon[25975]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:25.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:25.412 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[25136]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: Reconfiguring iscsi.foo.smithi027.temphd (dependencies changed)... 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: Reconfiguring daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:04:25.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:25 smithi027 ceph-mon[29814]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:27.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:26 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:27.225 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:26 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:27.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:26 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:28.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:27 smithi027 ceph-mon[25136]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:28.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:27 smithi027 ceph-mon[29814]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:28.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:27 smithi120 ceph-mon[25975]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:04:28.863 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:28] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-21T01:04:28.953 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:29.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all mgr 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all crash 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all mds 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all nfs 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1196954171' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[29814]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all mgr 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-21T01:04:29.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all crash 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all mds 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all nfs 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1196954171' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:29.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:29 smithi027 ceph-mon[25136]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:29.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all mgr 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-06-21T01:04:29.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all crash 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all mds 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all nfs 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1196954171' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:04:29.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:29 smithi120 ceph-mon[25975]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:29.645 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:04:29.645 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (13m) 27s ago 21m 22.2M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T01:04:29.645 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (13m) 9s ago 21m 54.6M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:04:29.645 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 starting - - - - 2024-06-21T01:04:29.645 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (14s) 9s ago 25m 53.5M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (10m) 27s ago 27m 476M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (27m) 27s ago 27m 75.9M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (26m) 9s ago 26m 60.5M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (26m) 27s ago 26m 60.1M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (13m) 27s ago 22m 16.3M - 1.3.1 1dbe0e931976 6cf10dcae6a7 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (13m) 9s ago 21m 16.7M - 1.3.1 1dbe0e931976 1d068cd1ae43 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (25m) 27s ago 25m 55.0M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (25m) 27s ago 25m 57.3M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (24m) 27s ago 24m 49.9M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (24m) 27s ago 24m 53.8M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (23m) 9s ago 23m 56.2M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (23m) 9s ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (23m) 9s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:04:29.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (22m) 9s ago 22m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:04:29.647 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (18s) 9s ago 22m 55.6M - 2.33.4 514e6a882f6e d7f5beab5bd1 2024-06-21T01:04:29.647 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (20m) 27s ago 20m 78.7M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:04:29.647 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (20m) 9s ago 20m 76.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:04:30.070 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:04:30.070 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:04:30.070 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:04:30.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:04:30.071 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:04:30.072 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:04:30.072 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:04:30.445 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:30 smithi027 systemd[1]: Stopping Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:30.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[29814]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[29814]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[29814]: Upgrade: Updating node-exporter.a (1/2) 2024-06-21T01:04:30.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[29814]: Deploying daemon node-exporter.a on smithi027 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1327748243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[25136]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[25136]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[25136]: Upgrade: Updating node-exporter.a (1/2) 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[25136]: Deploying daemon node-exporter.a on smithi027 2024-06-21T01:04:30.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:30 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1327748243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:30.447 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:04:30.447 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading node-exporter daemons", 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:04:30.448 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:04:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:30 smithi120 ceph-mon[25975]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:30 smithi120 ceph-mon[25975]: from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:30 smithi120 ceph-mon[25975]: Upgrade: Updating node-exporter.a (1/2) 2024-06-21T01:04:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:30 smithi120 ceph-mon[25975]: Deploying daemon node-exporter.a on smithi027 2024-06-21T01:04:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:30 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1327748243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:30.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:30 smithi027 podman[77708]: 2024-06-21 01:04:30.576192826 +0000 UTC m=+0.158848360 container died 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:31.228 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:30 smithi027 podman[77708]: 2024-06-21 01:04:30.940752913 +0000 UTC m=+0.523408445 container cleanup 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:31.228 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:30 smithi027 bash[77708]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-21T01:04:31.666 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 podman[77724]: 2024-06-21 01:04:31.250297783 +0000 UTC m=+0.677310925 container remove 6cf10dcae6a733bc5e3e7954cecbed2ff932ee98426c37e758ee5718d09c30b3 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:31.666 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-21T01:04:31.666 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-21T01:04:31.666 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 systemd[1]: Stopped Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:31.667 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 5.750s CPU time. 2024-06-21T01:04:31.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:31 smithi027 ceph-mon[29814]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:31.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:31 smithi027 ceph-mon[29814]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:31 smithi027 ceph-mon[25136]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:31.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:31 smithi027 ceph-mon[25136]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:31.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 systemd[1]: Starting Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:31.979 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:31 smithi027 bash[77849]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-06-21T01:04:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:31 smithi120 ceph-mon[25975]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:31 smithi120 ceph-mon[25975]: from='client.15621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:04:32.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:32 smithi027 bash[77849]: Getting image source signatures 2024-06-21T01:04:32.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:32 smithi027 bash[77849]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-06-21T01:04:32.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:32 smithi027 bash[77849]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-06-21T01:04:32.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:32 smithi027 bash[77849]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-06-21T01:04:33.861 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:33 smithi027 bash[77849]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-06-21T01:04:33.861 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:33 smithi027 bash[77849]: Writing manifest to image destination 2024-06-21T01:04:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:33 smithi120 ceph-mon[25975]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:34.133 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:33 smithi027 ceph-mon[25136]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:34.134 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:33 smithi027 ceph-mon[29814]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:34.134 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 podman[77849]: 2024-06-21 01:04:34.012737462 +0000 UTC m=+2.241605109 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 podman[77849]: 2024-06-21 01:04:34.13627973 +0000 UTC m=+2.365147414 container create 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 podman[77849]: 2024-06-21 01:04:34.372267931 +0000 UTC m=+2.601135590 container init 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 podman[77849]: 2024-06-21 01:04:34.375756678 +0000 UTC m=+2.604624326 container start 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.380Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.380Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.381Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.381Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.381Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=arp 2024-06-21T01:04:34.481 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=bcache 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=bonding 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=btrfs 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=conntrack 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=cpu 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=diskstats 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=dmi 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=edac 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=entropy 2024-06-21T01:04:34.482 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=filefd 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=filesystem 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=hwmon 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=infiniband 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=ipvs 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=loadavg 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=mdadm 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=meminfo 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=netclass 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=netdev 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=netstat 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=nfs 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=nfsd 2024-06-21T01:04:34.483 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=nvme 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=os 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=pressure 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=rapl 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=schedstat 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=selinux 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=sockstat 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=softnet 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=stat 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=tapestats 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=textfile 2024-06-21T01:04:34.484 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=time 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=uname 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=vmstat 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=xfs 2024-06-21T01:04:34.486 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=node_exporter.go:117 level=info collector=zfs 2024-06-21T01:04:34.487 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-06-21T01:04:34.487 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a[77898]: ts=2024-06-21T01:04:34.382Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-06-21T01:04:34.487 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 bash[77849]: 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 2024-06-21T01:04:34.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:04:34 smithi027 systemd[1]: Started Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:35.827 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:35 smithi120 ceph-mon[25975]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:35.827 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:35 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:35.828 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:35 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[25136]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:35.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[29814]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:35.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:35.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:35 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:36.589 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:36 smithi120 systemd[1]: Stopping Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:36.864 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:36 smithi120 podman[62086]: 2024-06-21 01:04:36.644316168 +0000 UTC m=+0.132588229 container died 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:36 smithi120 ceph-mon[25975]: Upgrade: Updating node-exporter.b (2/2) 2024-06-21T01:04:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:36 smithi120 ceph-mon[25975]: Deploying daemon node-exporter.b on smithi120 2024-06-21T01:04:37.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:36 smithi120 podman[62086]: 2024-06-21 01:04:36.99548052 +0000 UTC m=+0.483752580 container cleanup 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:37.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:36 smithi120 bash[62086]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-21T01:04:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:36 smithi027 ceph-mon[29814]: Upgrade: Updating node-exporter.b (2/2) 2024-06-21T01:04:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:36 smithi027 ceph-mon[29814]: Deploying daemon node-exporter.b on smithi120 2024-06-21T01:04:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:36 smithi027 ceph-mon[25136]: Upgrade: Updating node-exporter.b (2/2) 2024-06-21T01:04:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:36 smithi027 ceph-mon[25136]: Deploying daemon node-exporter.b on smithi120 2024-06-21T01:04:37.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 podman[62101]: 2024-06-21 01:04:37.268623042 +0000 UTC m=+0.627538668 container remove 1d068cd1ae43d073dd0a429ddb5aae76f487b1dac34439fd1571103a9c56d8ff (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:37.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-21T01:04:37.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-21T01:04:37.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 systemd[1]: Stopped Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:37.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 5.194s CPU time. 2024-06-21T01:04:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:37 smithi120 ceph-mon[25975]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:38.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 systemd[1]: Starting Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:04:38.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:37 smithi120 bash[62226]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-06-21T01:04:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:37 smithi027 ceph-mon[29814]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:37 smithi027 ceph-mon[25136]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:04:38.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:38 smithi120 bash[62226]: Getting image source signatures 2024-06-21T01:04:38.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:38 smithi120 bash[62226]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-06-21T01:04:38.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:38 smithi120 bash[62226]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-06-21T01:04:38.616 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:38 smithi120 bash[62226]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-06-21T01:04:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:38] "GET /metrics HTTP/1.1" 200 37913 "" "Prometheus/2.33.4" 2024-06-21T01:04:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:39 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:39 smithi027 ceph-mon[25136]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:40.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:39 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:39 smithi027 ceph-mon[29814]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:40.362 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:39 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:40.363 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:39 smithi120 ceph-mon[25975]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:40.363 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:39 smithi120 bash[62226]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-06-21T01:04:40.363 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 bash[62226]: Writing manifest to image destination 2024-06-21T01:04:40.618 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 podman[62226]: 2024-06-21 01:04:40.2872845 +0000 UTC m=+2.544535066 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-06-21T01:04:40.618 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 podman[62226]: 2024-06-21 01:04:40.385644523 +0000 UTC m=+2.642895094 container create b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:40.618 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 podman[62226]: 2024-06-21 01:04:40.570881429 +0000 UTC m=+2.828132008 container init b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 podman[62226]: 2024-06-21 01:04:40.573600507 +0000 UTC m=+2.830851076 container start b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.577Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.577Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.577Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.577Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=arp 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=bcache 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=bonding 2024-06-21T01:04:40.619 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=btrfs 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=conntrack 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=cpu 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=diskstats 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=dmi 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=edac 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=entropy 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=filefd 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=filesystem 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=hwmon 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=infiniband 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=ipvs 2024-06-21T01:04:40.620 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=loadavg 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=mdadm 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=meminfo 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=netclass 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=netdev 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=netstat 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=nfs 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=nfsd 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=nvme 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=os 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=pressure 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=rapl 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=schedstat 2024-06-21T01:04:40.621 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=selinux 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=sockstat 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=softnet 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=stat 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=tapestats 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=textfile 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=time 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=uname 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=vmstat 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=xfs 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=node_exporter.go:117 level=info collector=zfs 2024-06-21T01:04:40.622 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-06-21T01:04:40.623 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b[62276]: ts=2024-06-21T01:04:40.578Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-06-21T01:04:41.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 bash[62226]: b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f 2024-06-21T01:04:41.116 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:04:40 smithi120 systemd[1]: Started Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:04:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[29814]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[25136]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:42.040 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:41 smithi120 ceph-mon[25975]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:42.041 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:41 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:42.041 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:41 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:42.041 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:41 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:42.977 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:42 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:42 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:42.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:42 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:43 smithi120 ceph-mon[25975]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:43 smithi027 ceph-mon[25136]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:43 smithi027 ceph-mon[29814]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.086 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:45 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.142 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:46.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:47 smithi120 ceph-mon[25975]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:47 smithi027 ceph-mon[25136]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:48.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:47 smithi027 ceph-mon[29814]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:48.829 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:48] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:49.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:04:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:51 smithi120 ceph-mon[25975]: Upgrade: Updating prometheus.a 2024-06-21T01:04:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:51 smithi120 ceph-mon[25975]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:51 smithi120 ceph-mon[25975]: Deploying daemon prometheus.a on smithi120 2024-06-21T01:04:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[25136]: Upgrade: Updating prometheus.a 2024-06-21T01:04:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[25136]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[25136]: Deploying daemon prometheus.a on smithi120 2024-06-21T01:04:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[29814]: Upgrade: Updating prometheus.a 2024-06-21T01:04:52.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[29814]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:52.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:51 smithi027 ceph-mon[29814]: Deploying daemon prometheus.a on smithi120 2024-06-21T01:04:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:53 smithi120 ceph-mon[25975]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:53 smithi027 ceph-mon[25136]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:53 smithi027 ceph-mon[29814]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:04:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:55 smithi027 ceph-mon[25136]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:56.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:55 smithi027 ceph-mon[29814]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:56.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:55 smithi120 ceph-mon[25975]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:56 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:56 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:57.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:56 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:04:58.062 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:57 smithi120 ceph-mon[25975]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:58.236 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:57 smithi027 ceph-mon[25136]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:58.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:57 smithi027 ceph-mon[29814]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:04:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:04:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:04:58] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-06-21T01:05:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:59 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:04:59 smithi027 ceph-mon[25136]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:59 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:04:59 smithi027 ceph-mon[29814]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:00.276 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:59 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:00.276 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:04:59 smithi120 ceph-mon[25975]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:00.818 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:05:01.116 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:00 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:01.491 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (13m) 16s ago 21m 22.2M - 0.23.0 ba2b418f427c 117d76ebbaeb 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (13m) 16s ago 21m 54.8M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (33s) 16s ago 20m 50.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (45s) 16s ago 26m 438M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (10m) 16s ago 28m 493M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (28m) 16s ago 28m 76.7M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (26m) 16s ago 26m 57.5M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (26m) 16s ago 26m 60.4M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (27s) 16s ago 22m 8518k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:05:01.492 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (21s) 16s ago 22m 3338k - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (26m) 16s ago 26m 54.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (25m) 16s ago 25m 58.7M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (25m) 16s ago 25m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (24m) 16s ago 24m 54.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (24m) 16s ago 24m 56.3M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (24m) 16s ago 24m 53.0M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (23m) 16s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (23m) 16s ago 23m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (50s) 16s ago 22m 56.6M - 2.33.4 514e6a882f6e d7f5beab5bd1 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (21m) 16s ago 21m 79.1M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:05:01.493 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (21m) 16s ago 21m 77.0M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:05:01.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.343Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[60267]: ts=2024-06-21T01:05:01.344Z caller=main.go:1066 level=info msg="See you next time!" 2024-06-21T01:05:01.617 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 podman[63751]: 2024-06-21 01:05:01.354204475 +0000 UTC m=+0.450075336 container died d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:05:01.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:05:01.907 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:05:02.000 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-mon[25975]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:02.000 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:01 smithi120 ceph-mon[25975]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:02.000 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 podman[63751]: 2024-06-21 01:05:01.743073196 +0000 UTC m=+0.838944057 container cleanup d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:02.000 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:01 smithi120 bash[63751]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T01:05:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:01 smithi027 ceph-mon[25136]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:01 smithi027 ceph-mon[25136]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:01 smithi027 ceph-mon[29814]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:01 smithi027 ceph-mon[29814]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading prometheus daemons", 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:05:02.251 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:05:02.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 podman[63766]: 2024-06-21 01:05:02.002843891 +0000 UTC m=+0.648456791 container remove d7f5beab5bd162b40b17c972fbc699ee298911b1a594c0722e6b894c0c0db303 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:02.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-06-21T01:05:02.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 systemd[1]: Stopped Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:02.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service: Consumed 1.321s CPU time. 2024-06-21T01:05:02.635 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 systemd[1]: Starting Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:02.635 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 podman[63892]: 2024-06-21 01:05:02.494242169 +0000 UTC m=+0.017955221 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-06-21T01:05:02.635 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 podman[63892]: 2024-06-21 01:05:02.637927412 +0000 UTC m=+0.161640462 container create a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:02.896 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 podman[63892]: 2024-06-21 01:05:02.873563137 +0000 UTC m=+0.397276188 container init a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:02.896 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 podman[63892]: 2024-06-21 01:05:02.876675597 +0000 UTC m=+0.400388655 container start a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[25136]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[25136]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4171100567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[29814]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[29814]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:02 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4171100567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-mon[25975]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-mon[25975]: from='client.25543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/4171100567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:03.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.900Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.900Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.900Z caller=main.go:570 level=info host_details="(Linux 5.14.0-460.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Jun 6 12:49:42 UTC 2024 x86_64 smithi120 (none))" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.901Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.901Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.902Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.902Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.904Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.904Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.905Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.905Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.509µs 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.905Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-06-21T01:05:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.912Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.925Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.936Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.939Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.941Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.942Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.942Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.942Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.839µs wal_replay_duration=37.350516ms wbl_replay_duration=126ns total_replay_duration=37.470845ms 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.946Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.946Z caller=main.go:1029 level=info msg="TSDB started" 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.946Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.968Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.765836ms db_storage=759ns remote_storage=1.157µs web_handler=339ns query_engine=664ns scrape=400.679µs scrape_sd=87.915µs notify=14.945µs notify_sd=27.802µs rules=20.922855ms tracing=7.377µs 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.968Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-06-21T01:05:03.368 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:05:02.968Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-06-21T01:05:03.369 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 bash[63892]: a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 2024-06-21T01:05:03.369 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:05:02 smithi120 systemd[1]: Started Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[25136]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[25136]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[29814]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[29814]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.334 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.335 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:04 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:04.345 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:04 smithi120 ceph-mon[25975]: from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:04.345 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:04 smithi120 ceph-mon[25975]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:04.346 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:04 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.346 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:04 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:04.346 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:04 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:06 smithi120 ceph-mon[25975]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:06.315 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:06 smithi027 ceph-mon[25136]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:06.316 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:06 smithi027 ceph-mon[29814]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:07 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.262 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-06-21T01:05:08.262 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.263 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.263 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.263 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.263 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.263 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:07 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:08] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-06-21T01:05:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:09 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:09 smithi027 ceph-mon[25136]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:09 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:09 smithi027 ceph-mon[29814]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:09 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:09 smithi120 ceph-mon[25975]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:11.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:12.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:12 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:12.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:12 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:12 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:13.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[29814]: Upgrade: Updating alertmanager.a 2024-06-21T01:05:13.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[29814]: Deploying daemon alertmanager.a on smithi027 2024-06-21T01:05:13.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[29814]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[25136]: Upgrade: Updating alertmanager.a 2024-06-21T01:05:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[25136]: Deploying daemon alertmanager.a on smithi027 2024-06-21T01:05:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:13 smithi027 ceph-mon[25136]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:13 smithi120 ceph-mon[25975]: Upgrade: Updating alertmanager.a 2024-06-21T01:05:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:13 smithi120 ceph-mon[25975]: Deploying daemon alertmanager.a on smithi027 2024-06-21T01:05:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:13 smithi120 ceph-mon[25975]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:15.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:15 smithi027 ceph-mon[25136]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:15 smithi027 ceph-mon[29814]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:15 smithi120 ceph-mon[25975]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:17.965 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:17 smithi027 ceph-mon[25136]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:17.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:17 smithi027 ceph-mon[29814]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:17 smithi120 ceph-mon[25975]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:18.968 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:18] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.43.0" 2024-06-21T01:05:19.698 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:19 smithi027 systemd[1]: Stopping Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:19.698 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[59123]: level=info ts=2024-06-21T01:05:19.646Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:05:19.962 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:19 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:20.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:19 smithi027 ceph-mon[25136]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:20.020 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:19 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:20.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:19 smithi027 ceph-mon[29814]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:20.021 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:19 smithi027 podman[79987]: 2024-06-21 01:05:19.697805336 +0000 UTC m=+0.135957726 container died 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:20.228 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 podman[79987]: 2024-06-21 01:05:20.146955202 +0000 UTC m=+0.585107550 container cleanup 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:20.228 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 bash[79987]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-21T01:05:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:19 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:19 smithi120 ceph-mon[25975]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:20.612 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 podman[80002]: 2024-06-21 01:05:20.331785536 +0000 UTC m=+0.625918064 container remove 117d76ebbaeb1263394e888cd265a4d8846290fd01853b7862febf5d53b74e61 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:20.613 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 podman[80002]: 2024-06-21 01:05:20.40753302 +0000 UTC m=+0.701665559 volume remove 1788fbfcf5947efe53416d5f1a0bf3b71854904209cd7894439a3acc26f89cc5 2024-06-21T01:05:20.978 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-21T01:05:20.978 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 systemd[1]: Stopped Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:20.978 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.580s CPU time. 2024-06-21T01:05:20.978 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:20 smithi027 systemd[1]: Starting Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:21.322 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 podman[80131]: 2024-06-21 01:05:20.966049319 +0000 UTC m=+0.017639846 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-06-21T01:05:21.322 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 podman[80131]: 2024-06-21 01:05:21.109131556 +0000 UTC m=+0.160722083 volume create 1dc9e94ba6dabedf6c386af78097fedfe68345bad4101323988db9aa4744900f 2024-06-21T01:05:21.322 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 podman[80131]: 2024-06-21 01:05:21.184394324 +0000 UTC m=+0.235984836 container create 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 podman[80131]: 2024-06-21 01:05:21.428872461 +0000 UTC m=+0.480462985 container init 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 podman[80131]: 2024-06-21 01:05:21.431946726 +0000 UTC m=+0.483537239 container start 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.445Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.445Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.446Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-06-21T01:05:21.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.448Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.471Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.472Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.474Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:21.474Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 bash[80131]: 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 2024-06-21T01:05:21.729 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:21 smithi027 systemd[1]: Started Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:22.132 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[25136]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:22.133 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.133 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.133 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:22.133 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[29814]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:22.134 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.134 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.134 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:22.153 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:21 smithi120 ceph-mon[25975]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:22.153 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.153 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:22.154 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:21 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:23.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:23 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:23.448Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000829795s 2024-06-21T01:05:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:23 smithi120 ceph-mon[25975]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:24.137 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:23 smithi027 ceph-mon[25136]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:24.137 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:23 smithi027 ceph-mon[29814]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:25 smithi120 ceph-mon[25975]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:25 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[29814]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[25136]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:25 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[25136]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:27.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:27.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[29814]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:27.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:27 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:27.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:27 smithi120 ceph-mon[25975]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:27.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:27 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:27 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:27.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:27 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:28] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-06-21T01:05:29.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:29 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:29.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:29 smithi027 ceph-mon[29814]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:29.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:29 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:29.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:29 smithi027 ceph-mon[25136]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:29 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:29 smithi120 ceph-mon[25975]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.056 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:30 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-21T01:05:31.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:30 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:31.728 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:05:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:05:31.452Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004070536s 2024-06-21T01:05:32.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:31 smithi027 ceph-mon[25136]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:31 smithi027 ceph-mon[29814]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:31 smithi120 ceph-mon[25975]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:32.617 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:05:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:32 smithi027 ceph-mon[25136]: Upgrade: Updating grafana.a 2024-06-21T01:05:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:32 smithi027 ceph-mon[29814]: Upgrade: Updating grafana.a 2024-06-21T01:05:33.290 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (12s) 7s ago 22m 17.2M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (14m) 7s ago 22m 54.5M - 8.3.5 dad864ee21e9 1da7e7d8e2e3 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (65s) 7s ago 21m 50.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (77s) 7s ago 26m 438M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (11m) 7s ago 28m 497M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (28m) 7s ago 28m 77.4M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (27m) 7s ago 27m 57.5M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (27m) 7s ago 27m 60.4M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (59s) 7s ago 23m 8120k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (52s) 7s ago 23m 15.2M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (26m) 7s ago 26m 54.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (26m) 7s ago 26m 58.7M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:05:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (25m) 7s ago 25m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (25m) 7s ago 25m 54.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (25m) 7s ago 25m 56.3M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (24m) 7s ago 24m 53.0M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (24m) 7s ago 24m 52.8M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (23m) 7s ago 23m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (30s) 7s ago 23m 52.8M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (21m) 7s ago 21m 79.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:05:33.292 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (21m) 7s ago 21m 77.1M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:05:33.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:32 smithi120 ceph-mon[25975]: Upgrade: Updating grafana.a 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:05:33.702 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:05:33.703 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:05:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[25136]: Deploying daemon grafana.a on smithi120 2024-06-21T01:05:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[25136]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[25136]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[25136]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3910304644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[29814]: Deploying daemon grafana.a on smithi120 2024-06-21T01:05:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[29814]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[29814]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[29814]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:33.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:33 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3910304644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:05:34.050 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-06-21T01:05:34.051 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading grafana daemons", 2024-06-21T01:05:34.051 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:05:34.051 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:05:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:33 smithi120 ceph-mon[25975]: Deploying daemon grafana.a on smithi120 2024-06-21T01:05:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:33 smithi120 ceph-mon[25975]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:33 smithi120 ceph-mon[25975]: from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:33 smithi120 ceph-mon[25975]: from='client.15663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:33 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3910304644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:34 smithi027 ceph-mon[25136]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:34 smithi027 ceph-mon[29814]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:35.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:34 smithi120 ceph-mon[25975]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:35 smithi027 ceph-mon[25136]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:35 smithi027 ceph-mon[25136]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:36.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:35 smithi027 ceph-mon[29814]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:36.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:35 smithi027 ceph-mon[29814]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:35 smithi120 ceph-mon[25975]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:05:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:35 smithi120 ceph-mon[25975]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:37 smithi027 ceph-mon[25136]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:37 smithi027 ceph-mon[29814]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:38.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:37 smithi120 ceph-mon[25975]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:38] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.43.0" 2024-06-21T01:05:40.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:39 smithi120 ceph-mon[25975]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:40.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:39 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:39 smithi027 ceph-mon[25136]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:39 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:39 smithi027 ceph-mon[29814]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:39 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:41.892 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:41 smithi120 systemd[1]: Stopping Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:41.892 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[49606]: t=2024-06-21T01:05:41+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-06-21T01:05:41.892 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:41 smithi120 podman[65948]: 2024-06-21 01:05:41.880076539 +0000 UTC m=+0.095923273 container died 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Paul Cuzner , release=236.1648460182, name=ubi8, io.openshift.expose-services=, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8.5, build-date=2022-03-28T10:36:18.413762) 2024-06-21T01:05:42.183 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:41 smithi120 ceph-mon[25975]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:42.184 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:41 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:41 smithi027 ceph-mon[29814]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:42.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:41 smithi027 ceph-mon[25136]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:42.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:42.546 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 podman[65948]: 2024-06-21 01:05:42.186691875 +0000 UTC m=+0.402538610 container cleanup 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base rhel8, release=236.1648460182, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, maintainer=Paul Cuzner , name=ubi8, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2) 2024-06-21T01:05:42.546 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 bash[65948]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a 2024-06-21T01:05:42.547 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 podman[65960]: 2024-06-21 01:05:42.421432689 +0000 UTC m=+0.542407249 container remove 1da7e7d8e2e35c76040fcb4e90a92cd06b14d8a9489a148a0963d5d7a77abf79 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, name=ubi8, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, distribution-scope=public, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , architecture=x86_64, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.openshift.tags=base rhel8) 2024-06-21T01:05:42.866 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-21T01:05:42.866 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 systemd[1]: Stopped Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:42.866 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 5.489s CPU time. 2024-06-21T01:05:42.866 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 systemd[1]: Starting Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:05:43.231 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 podman[66087]: 2024-06-21 01:05:42.873777764 +0000 UTC m=+0.017928579 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-06-21T01:05:43.231 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:42 smithi120 podman[66087]: 2024-06-21 01:05:42.981409245 +0000 UTC m=+0.125560044 container create 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:05:43.231 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 podman[66087]: 2024-06-21 01:05:43.150584478 +0000 UTC m=+0.294735291 container init 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:05:43.231 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 podman[66087]: 2024-06-21 01:05:43.155361104 +0000 UTC m=+0.299511905 container start 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:05:43.481 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 bash[66087]: 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 systemd[1]: Started Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302438911Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.30265438Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302667951Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.30267353Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.30267812Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.3026833Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302687768Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302693018Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-06-21T01:05:43.482 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302698017Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302703052Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302709241Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302713941Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302718705Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.30272346Z level=info msg="Path Home" path=/usr/share/grafana 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302727877Z level=info msg="Path Data" path=/var/lib/grafana 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302732096Z level=info msg="Path Logs" path=/var/log/grafana 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302736541Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302741167Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=settings t=2024-06-21T01:05:43.302745806Z level=info msg="App mode production" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=sqlstore t=2024-06-21T01:05:43.302795776Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=sqlstore t=2024-06-21T01:05:43.302811807Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.31443194Z level=info msg="Starting DB migrations" 2024-06-21T01:05:43.483 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.31795717Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.329401818Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.330596327Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.333340137Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.336102335Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.338685324Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.341361945Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.344089019Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.346802346Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.349527355Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.349907Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.352519342Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.355192918Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-06-21T01:05:43.484 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.357791618Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-06-21T01:05:43.485 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.358882399Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-06-21T01:05:43.485 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.361601088Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-06-21T01:05:43.485 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.364367863Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-06-21T01:05:43.485 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.365470102Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.366581852Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.367359122Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.36845759Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.369514066Z level=info msg="Executing migration" id="create provenance_type table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.369961775Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.370841345Z level=info msg="Executing migration" id="create alert_image table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.371648909Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.372431522Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.372736837Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.37358015Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.374399189Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.374481263Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-06-21T01:05:43.486 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.375210302Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.376024402Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.376877159Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.377755435Z level=info msg="Executing migration" id="create secrets table" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.378506988Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.384740449Z level=info msg="Executing migration" id="add name column into data_keys" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.385926898Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.386778212Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.393134718Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.399429543Z level=info msg="Executing migration" id="add column hidden to role table" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.400580464Z level=info msg="Executing migration" id="create query_history table v1" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.40145598Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.40227341Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.402567337Z level=info msg="Executing migration" id="teams permissions migration" 2024-06-21T01:05:43.487 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.403425147Z level=info msg="Executing migration" id="dashboard permissions" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.404660721Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.405325382Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.406129139Z level=info msg="Executing migration" id="alerting notification permissions" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.406474989Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.407345621Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.408163843Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.40932253Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.409632916Z level=info msg="Executing migration" id="create correlation table v1" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.412598803Z level=info msg="Executing migration" id="add index correlations.uid" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.415361758Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.417955733Z level=info msg="Executing migration" id="add correlation config column" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.419098862Z level=info msg="Executing migration" id="create entity_events table" 2024-06-21T01:05:43.488 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.421827574Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.424731989Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.424810853Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.427667188Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.42774081Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.42804573Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.430751475Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.433576094Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.436450692Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.436900703Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.439590956Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.442362199Z level=info msg="Executing migration" id="Drop public config table" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.44515458Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-06-21T01:05:43.489 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.445608085Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.448267854Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.448737304Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.451072719Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.454851528Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.456045888Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.458739716Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.461482757Z level=info msg="Executing migration" id="add share column" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.464165281Z level=info msg="Executing migration" id="create default alerting folders" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.464623192Z level=info msg="Executing migration" id="create file table" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.467252067Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.469895602Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-06-21T01:05:43.490 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.472691975Z level=info msg="Executing migration" id="create file_meta table" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.473090201Z level=info msg="Executing migration" id="file table idx: path key" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.475746749Z level=info msg="Executing migration" id="set path collation in file table" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.478371448Z level=info msg="Executing migration" id="managed permissions migration" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.481064165Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.481632905Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.484239652Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-06-21T01:05:43.491 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.486762443Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.487635604Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.488092771Z level=info msg="Executing migration" id="update group index for alert rules" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.488872284Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.489668992Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.490497912Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.491847222Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.493380921Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.494152321Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-06-21T01:05:43.837 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.508259354Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.508999637Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.509724577Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.514226655Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.514995602Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.515850833Z level=info msg="Executing migration" id="create folder table" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.516702405Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.51756338Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.518376351Z level=info msg="Executing migration" id="Update folder title length" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.519268724Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=migrator t=2024-06-21T01:05:43.520123965Z level=info msg="migrations completed" performed=105 skipped=376 duration=202.213667ms 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=sqlstore t=2024-06-21T01:05:43.520445097Z level=info msg="Created default organization" 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=plugin.finder t=2024-06-21T01:05:43.553953386Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=secrets t=2024-06-21T01:05:43.554052512Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-06-21T01:05:43.838 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=query_data t=2024-06-21T01:05:43.558110462Z level=info msg="Query Service initialization" 2024-06-21T01:05:43.839 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=live.push_http t=2024-06-21T01:05:43.560876775Z level=info msg="Live Push Gateway initialization" 2024-06-21T01:05:44.094 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-mon[25975]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:44.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.095 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:43 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[25136]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[29814]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:43 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:44.361 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=infra.usagestats.collector t=2024-06-21T01:05:44.197768525Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-06-21T01:05:44.361 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=provisioning.datasources t=2024-06-21T01:05:44.199610591Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=provisioning.datasources t=2024-06-21T01:05:44.199619436Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=provisioning.datasources t=2024-06-21T01:05:44.1997129Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=provisioning.alerting t=2024-06-21T01:05:44.22132919Z level=info msg="starting to provision alerting" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=provisioning.alerting t=2024-06-21T01:05:44.221338822Z level=info msg="finished to provision alerting" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=ngalert.state.manager t=2024-06-21T01:05:44.221412399Z level=info msg="Warming state cache for startup" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=grafanaStorageLogger t=2024-06-21T01:05:44.221455616Z level=info msg="storage starting" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=http.server t=2024-06-21T01:05:44.224276111Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=ngalert.state.manager t=2024-06-21T01:05:44.23169167Z level=info msg="State cache has been initialized" states=0 duration=10.276215ms 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=ticker t=2024-06-21T01:05:44.231850957Z level=info msg=starting first_tick=2024-06-21T01:05:50Z 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=ngalert.multiorg.alertmanager t=2024-06-21T01:05:44.23190214Z level=info msg="starting MultiOrg Alertmanager" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=sqlstore.transactions t=2024-06-21T01:05:44.245257924Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-06-21T01:05:44.362 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:05:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=sqlstore.transactions t=2024-06-21T01:05:44.25586386Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-06-21T01:05:46.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:45 smithi027 ceph-mon[29814]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:46.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:45 smithi027 ceph-mon[25136]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:46.245 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:45 smithi120 ceph-mon[25975]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[25136]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[29814]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:47 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:47 smithi120 ceph-mon[25975]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:47 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:47 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:48] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.43.0" 2024-06-21T01:05:49.609 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.609 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:49 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.609 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:49 smithi120 ceph-mon[25975]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:49.609 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:49 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:49.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[29814]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:49.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:49.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:49.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[25136]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:49.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:49 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:05:51.876 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:05:51.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:05:51.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:05:52.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:05:52.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:05:52.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:05:52.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:05:52.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:05:52.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:05:52.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:51 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:52.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:05:52.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:05:52.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:05:52.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:52.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:51 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:05:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[25136]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:53.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[25136]: Upgrade: Finalizing container_image settings 2024-06-21T01:05:53.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[25136]: Upgrade: Complete! 2024-06-21T01:05:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[29814]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[29814]: Upgrade: Finalizing container_image settings 2024-06-21T01:05:53.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:52 smithi027 ceph-mon[29814]: Upgrade: Complete! 2024-06-21T01:05:53.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:52 smithi120 ceph-mon[25975]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-21T01:05:53.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:52 smithi120 ceph-mon[25975]: Upgrade: Finalizing container_image settings 2024-06-21T01:05:53.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:52 smithi120 ceph-mon[25975]: Upgrade: Complete! 2024-06-21T01:05:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:53 smithi027 ceph-mon[25136]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:54.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:53 smithi027 ceph-mon[29814]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:53 smithi120 ceph-mon[25975]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:05:56.060 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:55 smithi120 ceph-mon[25975]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:55 smithi027 ceph-mon[25136]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:56.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:55 smithi027 ceph-mon[29814]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:56 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:56 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:56 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:05:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:57 smithi027 ceph-mon[25136]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:57 smithi027 ceph-mon[29814]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:58.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:57 smithi120 ceph-mon[25975]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:05:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:05:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:05:58] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.43.0" 2024-06-21T01:06:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:59 smithi027 ceph-mon[25136]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:05:59 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:59 smithi027 ceph-mon[29814]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:05:59 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:59 smithi120 ceph-mon[25975]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:05:59 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:01 smithi027 ceph-mon[25136]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:01 smithi027 ceph-mon[29814]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:02.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:01 smithi120 ceph-mon[25975]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:03 smithi027 ceph-mon[25136]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:04.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:03 smithi027 ceph-mon[29814]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:03 smithi120 ceph-mon[25975]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:05.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-06-21T01:06:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:05 smithi027 ceph-mon[25136]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:05 smithi027 ceph-mon[25136]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:06.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:05 smithi027 ceph-mon[29814]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:06.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:05 smithi027 ceph-mon[29814]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:06.292 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:05 smithi120 ceph-mon[25975]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:06.292 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:05 smithi120 ceph-mon[25975]: from='client.15687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:07.448 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:06:08.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:07 smithi027 ceph-mon[25136]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:08.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:07 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/266264930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:08.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:07 smithi027 ceph-mon[29814]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:08.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:07 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/266264930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:08.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-06-21T01:06:08.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:07 smithi120 ceph-mon[25975]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:08.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:07 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/266264930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:08] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.43.0" 2024-06-21T01:06:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:09 smithi027 ceph-mon[25136]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:09 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:09 smithi027 ceph-mon[29814]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:10.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:09 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:09 smithi120 ceph-mon[25975]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:09 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:10.449 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)" 2024-06-21T01:06:11.158 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:10 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/2152135639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:11.158 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:10 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2152135639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:11.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-06-21T01:06:11.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:10 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/2152135639' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:12.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:11 smithi027 ceph-mon[25136]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:12.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:12.180 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:11 smithi027 ceph-mon[29814]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:12.180 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:12.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:11 smithi120 ceph-mon[25975]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:12.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:11 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:13.482 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:06:14.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-06-21T01:06:14.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:13 smithi027 ceph-mon[25136]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:14.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:13 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3101282013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:14.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:13 smithi027 ceph-mon[29814]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:14.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:13 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3101282013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:14.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:13 smithi120 ceph-mon[25975]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:14.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:13 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/3101282013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:15 smithi027 ceph-mon[25136]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:15 smithi027 ceph-mon[29814]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:15 smithi120 ceph-mon[25975]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:18.017 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:06:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:17 smithi027 ceph-mon[25136]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:17 smithi027 ceph-mon[25136]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:17 smithi027 ceph-mon[29814]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:17 smithi027 ceph-mon[29814]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:17 smithi120 ceph-mon[25975]: from='client.15711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:17 smithi120 ceph-mon[25975]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:18.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:06:18.947 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:18] "GET /metrics HTTP/1.1" 200 37919 "" "Prometheus/2.43.0" 2024-06-21T01:06:20.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:19 smithi027 ceph-mon[25136]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:20.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:19 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:19 smithi027 ceph-mon[29814]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:19 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:19 smithi120 ceph-mon[25975]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:19 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:06:20.534 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:06:20.535 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:06:21.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:06:21.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:21 smithi027 ceph-mon[25136]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:21.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:21 smithi027 ceph-mon[25136]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:21.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:21 smithi027 ceph-mon[29814]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:21.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:21 smithi027 ceph-mon[29814]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:21 smithi120 ceph-mon[25975]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:21 smithi120 ceph-mon[25975]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:23.072 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:06:23.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-06-21T01:06:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:23 smithi027 ceph-mon[25136]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:23 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/250706567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:06:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:23 smithi027 ceph-mon[29814]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:24.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:23 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/250706567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:06:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:23 smithi120 ceph-mon[25975]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:23 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/250706567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:06:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:25 smithi027 ceph-mon[25136]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:26.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:25 smithi027 ceph-mon[29814]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:25 smithi120 ceph-mon[25975]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[25136]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[25136]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi120", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[29814]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[29814]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi120", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:26 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:27.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:26 smithi120 ceph-mon[25975]: from='client.25633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:26 smithi120 ceph-mon[25975]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi120", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:27.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:26 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:27 smithi027 ceph-mon[25136]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:27 smithi027 ceph-mon[29814]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:28.325 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:27 smithi120 ceph-mon[25975]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:28.603 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:28.908 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:28] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.43.0" 2024-06-21T01:06:29.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:29.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:29.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:29.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:29 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:29.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:29.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:30.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:30 smithi027 ceph-mon[25136]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:30.864 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:30 smithi027 ceph-mon[29814]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:30.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:30 smithi120 ceph-mon[25975]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:06:31.503 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:06:32.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:31 smithi027 ceph-mon[25136]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:32.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:31 smithi027 ceph-mon[29814]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:32.199 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:06:32.199 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (71s) 66s ago 23m 17.2M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:06:32.199 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (49s) 44s ago 23m 35.0M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:06:32.199 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 66s ago 22m 50.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (2m) 44s ago 27m 439M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (12m) 66s ago 29m 497M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (29m) 66s ago 29m 77.4M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (28m) 44s ago 28m 55.9M 2048M 17.2.0 e1d6a67b021e f3b8a281e7fa 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (28m) 66s ago 28m 60.4M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (118s) 66s ago 24m 8120k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (111s) 44s ago 24m 15.9M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (27m) 66s ago 27m 54.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (27m) 66s ago 27m 58.7M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (26m) 66s ago 26m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (26m) 66s ago 26m 54.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:06:32.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (26m) 44s ago 26m 56.3M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (25m) 44s ago 25m 53.0M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (25m) 44s ago 25m 52.8M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (24m) 44s ago 24m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (89s) 44s ago 24m 53.3M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (22m) 66s ago 22m 79.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:06:32.201 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (22m) 44s ago 22m 77.2M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:06:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:31 smithi120 ceph-mon[25975]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:06:32.611 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:06:32.612 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi120", 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/1 daemons upgraded", 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:06:32.960 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.028 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:06:33.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-21T01:06:33.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:32 smithi120 ceph-mon[25975]: from='client.? 172.21.15.27:0/1049613909' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/1049613909' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='client.15741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='client.25651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:33.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:06:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-06-21T01:06:33.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:32 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/1049613909' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: Upgrade: Setting container_image for all mgr 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: Upgrade: It appears safe to stop mon.b 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:06:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:06:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:33 smithi120 ceph-mon[25975]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all mgr 2024-06-21T01:06:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: Upgrade: It appears safe to stop mon.b 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='client.15753 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all mgr 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: Upgrade: It appears safe to stop mon.b 2024-06-21T01:06:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:06:34.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:34.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:06:34.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:06:34.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:33 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:34 smithi120 ceph-mon[25975]: Upgrade: Updating mon.b 2024-06-21T01:06:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:34 smithi120 ceph-mon[25975]: Deploying daemon mon.b on smithi120 2024-06-21T01:06:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:34 smithi027 ceph-mon[25136]: Upgrade: Updating mon.b 2024-06-21T01:06:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:34 smithi027 ceph-mon[25136]: Deploying daemon mon.b on smithi120 2024-06-21T01:06:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:34 smithi027 ceph-mon[29814]: Upgrade: Updating mon.b 2024-06-21T01:06:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:34 smithi027 ceph-mon[29814]: Deploying daemon mon.b on smithi120 2024-06-21T01:06:35.655 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:35 smithi120 systemd[1]: Stopping Ceph mon.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:06:36.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:35 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b[25968]: 2024-06-21T01:06:35.656+0000 7f6794090700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:06:36.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:35 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b[25968]: 2024-06-21T01:06:35.656+0000 7f6794090700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-06-21T01:06:36.080 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:35 smithi120 podman[67553]: 2024-06-21 01:06:35.820285601 +0000 UTC m=+0.267831660 container died f3b8a281e7fa6496c5ad6bf4bc1cad678acb4d89966ec1d72d39be6c70785dd5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, vcs-type=git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True) 2024-06-21T01:06:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 podman[67553]: 2024-06-21 01:06:36.190387572 +0000 UTC m=+0.637933629 container cleanup f3b8a281e7fa6496c5ad6bf4bc1cad678acb4d89966ec1d72d39be6c70785dd5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, io.k8s.display-name=CentOS Stream 8, vcs-type=git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, ceph=True, vendor=Red Hat, Inc.) 2024-06-21T01:06:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 bash[67553]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b 2024-06-21T01:06:36.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 podman[67567]: 2024-06-21 01:06:36.475682722 +0000 UTC m=+0.649847942 container remove f3b8a281e7fa6496c5ad6bf4bc1cad678acb4d89966ec1d72d39be6c70785dd5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, com.redhat.component=centos-stream-container, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.expose-services=, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-06-21T01:06:36.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.b.service: Deactivated successfully. 2024-06-21T01:06:36.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 systemd[1]: Stopped Ceph mon.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:06:36.796 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.b.service: Consumed 17.152s CPU time. 2024-06-21T01:06:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:36 smithi120 systemd[1]: Starting Ceph mon.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:06:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 podman[67697]: 2024-06-21 01:06:36.916795384 +0000 UTC m=+0.017634624 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:06:37.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 podman[67697]: 2024-06-21 01:06:37.043665032 +0000 UTC m=+0.144504309 container create 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:06:37.402 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 podman[67697]: 2024-06-21 01:06:37.296399439 +0000 UTC m=+0.397238684 container init 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618) 2024-06-21T01:06:37.402 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 podman[67697]: 2024-06-21 01:06:37.299839987 +0000 UTC m=+0.400679234 container start 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, io.buildah.version=1.36.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD) 2024-06-21T01:06:37.402 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: set uid:gid to 167:167 (ceph:ceph) 2024-06-21T01:06:37.402 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc), process ceph-mon, pid 2 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: pidfile_write: ignore empty --pid-file 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: load: jerasure load: lrc 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: RocksDB version: 7.9.2 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Git sha 0 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Compile date 2024-06-19 15:40:36 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: DB SUMMARY 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: DB Session ID: F4IS4ZNNE8B3X6M5OKKS 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: CURRENT file: CURRENT 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: IDENTITY file: IDENTITY 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2813 Bytes 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 2005687 ; 2024-06-21T01:06:37.403 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.error_if_exists: 0 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.create_if_missing: 0 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.paranoid_checks: 1 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.env: 0x55723d894740 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.fs: PosixFileSystem 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.info_log: 0x55723fc5d0c0 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_file_opening_threads: 16 2024-06-21T01:06:37.404 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.statistics: (nil) 2024-06-21T01:06:37.405 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.use_fsync: 0 2024-06-21T01:06:37.405 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_log_file_size: 0 2024-06-21T01:06:37.405 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.keep_log_file_num: 1000 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.recycle_log_file_num: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_fallocate: 1 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_mmap_reads: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_mmap_writes: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.use_direct_reads: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.create_missing_column_families: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.db_log_dir: 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.wal_dir: 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-21T01:06:37.406 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.advise_random_on_open: 1 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.db_write_buffer_size: 0 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.write_buffer_manager: 0x55723fc6cc80 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.rate_limiter: (nil) 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.wal_recovery_mode: 2 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enable_thread_tracking: 0 2024-06-21T01:06:37.407 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enable_pipelined_write: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.unordered_write: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.row_cache: None 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.wal_filter: None 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_ingest_behind: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.two_write_queues: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.manual_wal_flush: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.wal_compression: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.atomic_flush: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-21T01:06:37.408 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.log_readahead_size: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.best_efforts_recovery: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.allow_data_in_errors: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.db_host_id: __hostname__ 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_background_jobs: 2 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_background_compactions: -1 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_subcompactions: 1 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-21T01:06:37.409 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_total_wal_size: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_open_files: -1 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bytes_per_sync: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_readahead_size: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_background_flushes: -1 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Compression algorithms supported: 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kZSTD supported: 0 2024-06-21T01:06:37.410 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kXpressCompression supported: 0 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kBZip2Compression supported: 0 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kLZ4Compression supported: 1 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kZlibCompression supported: 1 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kLZ4HCCompression supported: 1 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: kSnappyCompression supported: 1 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.merge_operator: 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_filter: None 2024-06-21T01:06:37.411 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_filter_factory: None 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.sst_partitioner_factory: None 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55723fc5cd20) 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: cache_index_and_filter_blocks: 1 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: pin_top_level_index_and_filter: 1 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: index_type: 0 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: data_block_index_type: 0 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: index_shortening: 1 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: checksum: 4 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: no_block_cache: 0 2024-06-21T01:06:37.412 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_cache: 0x55723fc551f0 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_cache_name: BinnedLRUCache 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_cache_options: 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: capacity : 536870912 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: num_shard_bits : 4 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: strict_capacity_limit : 0 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: high_pri_pool_ratio: 0.000 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_cache_compressed: (nil) 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: persistent_cache: (nil) 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_size: 4096 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_size_deviation: 10 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_restart_interval: 16 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: index_block_restart_interval: 1 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: metadata_block_size: 4096 2024-06-21T01:06:37.413 INFO:journalctl@ceph.mon.b.smithi120.stdout: partition_filters: 0 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: use_delta_encoding: 1 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: filter_policy: bloomfilter 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: whole_key_filtering: 1 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: verify_compression: 0 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: read_amp_bytes_per_bit: 0 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: format_version: 5 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: enable_index_compression: 1 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: block_align: 0 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: max_auto_readahead_size: 262144 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: prepopulate_block_cache: 0 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: initial_auto_readahead_size: 8192 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout: num_file_reads_for_auto_readahead: 2 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.write_buffer_size: 33554432 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_write_buffer_number: 2 2024-06-21T01:06:37.414 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression: NoCompression 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression: Disabled 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.prefix_extractor: nullptr 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.num_levels: 7 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-21T01:06:37.415 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.level: 32767 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.strategy: 0 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.enabled: false 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-21T01:06:37.416 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.target_file_size_base: 67108864 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-21T01:06:37.417 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.arena_block_size: 1048576 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.disable_auto_compactions: 0 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-21T01:06:37.418 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.table_properties_collectors: 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.inplace_update_support: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.bloom_locality: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.max_successive_merges: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.paranoid_file_checks: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.force_consistency_checks: 1 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.report_bg_io_stats: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.ttl: 2592000 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-21T01:06:37.419 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enable_blob_files: false 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.min_blob_size: 0 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_file_size: 268435456 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.blob_file_starting_level: 0 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:06:37.420 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28843, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3d0c3182-d187-401f-9d81-02cd14df7b23 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718931997327499, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718931997335093, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1243726, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28840, "largest_seqno": 30078, "table_properties": {"data_size": 1238712, "index_size": 2784, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 11892, "raw_average_key_size": 24, "raw_value_size": 1228473, "raw_average_value_size": 2532, "num_data_blocks": 125, "num_entries": 485, "num_filter_entries": 485, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718931997, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3d0c3182-d187-401f-9d81-02cd14df7b23", "db_session_id": "F4IS4ZNNE8B3X6M5OKKS", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718931997335206, "job": 1, "event": "recovery_finished"} 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55723fc80e00 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: DB pointer 0x55723fd76000 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-21T01:06:37.421 INFO:journalctl@ceph.mon.b.smithi120.stdout: ** DB Stats ** 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: ** Compaction Stats [default] ** 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: L0 1/0 1.19 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 335.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: L6 1/0 9.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Sum 2/0 10.38 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 335.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:06:37.422 INFO:journalctl@ceph.mon.b.smithi120.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 335.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: ** Compaction Stats [default] ** 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 335.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T01:06:37.423 INFO:journalctl@ceph.mon.b.smithi120.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: Cumulative compaction: 0.00 GB write, 112.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: Interval compaction: 0.00 GB write, 112.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: Block cache BinnedLRUCache@0x55723fc551f0#2 capacity: 512.00 MB usage: 4.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: starting mon.b rank 2 at public addrs [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] at bind addrs [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???) e3 preinit fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:06:37.424 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).mds e1 new map 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).mds e1 print_map 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: e1 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: legacy client fscid: -1 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout: No filesystems configured 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 ceph-mon[67716]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 bash[67697]: 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a 2024-06-21T01:06:37.425 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:37 smithi120 systemd[1]: Started Ceph mon.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:06:38.640 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: mon.b calling monitor election 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: mon.c calling monitor election 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: mon.a calling monitor election 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: fsmap 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: mon.b calling monitor election 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: mon.c calling monitor election 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: mon.a calling monitor election 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: fsmap 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: overall HEALTH_OK 2024-06-21T01:06:38.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:38.805 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:38.805 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: mon.b calling monitor election 2024-06-21T01:06:38.805 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: mon.c calling monitor election 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: mon.a calling monitor election 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: fsmap 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: mgrmap e41: y(active, since 2m), standbys: x 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: overall HEALTH_OK 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:38.806 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:38 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:38] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.43.0" 2024-06-21T01:06:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:39 smithi120 ceph-mon[67716]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:39 smithi027 ceph-mon[29814]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:39 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:39 smithi027 ceph-mon[25136]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:39 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[25136]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[29814]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:41 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:41 smithi120 ceph-mon[67716]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:41 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:41 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:41 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:42.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:06:42.345+0000 7fbe19754640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-06-21T01:06:43.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[25136]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:43.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:43 smithi027 ceph-mon[29814]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:43.531 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:43 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.531 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:43 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:43.531 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:43 smithi120 ceph-mon[67716]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:45.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:45 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:45.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:45.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:45.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:45.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:45 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Detected new or changed devices on smithi120 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all crash 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all mds 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all nfs 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:06:46.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi120 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all crash 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all mds 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:06:46.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all nfs 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:06:46.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all grafana 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all loki 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all promtail 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Finalizing container_image settings 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:06:46.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all grafana 2024-06-21T01:06:46.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all loki 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all promtail 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Finalizing container_image settings 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:06:46.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:06:46.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: Upgrade: Complete! 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: Upgrade: Complete! 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:46 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Detected new or changed devices on smithi120 2024-06-21T01:06:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mds 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nfs 2024-06-21T01:06:46.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:06:46.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all grafana 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all loki 2024-06-21T01:06:46.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all promtail 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:06:46.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:06:46.873 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:06:46.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:06:46.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:46 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:47 smithi027 ceph-mon[25136]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:47 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:48.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:47 smithi027 ceph-mon[29814]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:48.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:47 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:47 smithi120 ceph-mon[67716]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:47 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:06:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:48] "GET /metrics HTTP/1.1" 200 37854 "" "Prometheus/2.43.0" 2024-06-21T01:06:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:49 smithi120 ceph-mon[67716]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:49 smithi027 ceph-mon[25136]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:49 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:49 smithi027 ceph-mon[29814]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:49 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:06:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:51 smithi027 ceph-mon[25136]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:51 smithi027 ceph-mon[29814]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:51 smithi120 ceph-mon[67716]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:53 smithi027 ceph-mon[25136]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:54.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:53 smithi027 ceph-mon[29814]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:53 smithi120 ceph-mon[67716]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:06:56.168 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:55 smithi120 ceph-mon[67716]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:55 smithi027 ceph-mon[25136]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:56.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:55 smithi027 ceph-mon[29814]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:56 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:56 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:57.332 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:56 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:06:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:57 smithi027 ceph-mon[25136]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:58.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:57 smithi027 ceph-mon[29814]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:58.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:57 smithi120 ceph-mon[67716]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:06:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:06:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:06:58] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-06-21T01:07:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:59 smithi027 ceph-mon[25136]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:00.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:06:59 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:59 smithi027 ceph-mon[29814]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:06:59 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:59 smithi120 ceph-mon[67716]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:06:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:01 smithi027 ceph-mon[25136]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:01 smithi027 ceph-mon[29814]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:02.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:01 smithi120 ceph-mon[67716]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:04.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:03 smithi027 ceph-mon[25136]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:03 smithi027 ceph-mon[29814]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:04.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:07:04.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:03 smithi120 ceph-mon[67716]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:04.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:04 smithi027 ceph-mon[25136]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:04.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:04 smithi027 ceph-mon[29814]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:05.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:04 smithi120 ceph-mon[67716]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (105s) 100s ago 23m 17.2M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (83s) 24s ago 23m 79.1M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 100s ago 22m 50.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (2m) 24s ago 28m 440M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (13m) 100s ago 30m 497M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:07:06.149 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (30m) 100s ago 30m 77.4M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (29s) 24s ago 28m 18.6M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (28m) 100s ago 28m 60.4M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 100s ago 24m 8120k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (2m) 24s ago 24m 15.5M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (28m) 100s ago 28m 54.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (27m) 100s ago 27m 58.7M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (27m) 100s ago 27m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (26m) 100s ago 26m 54.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (26m) 24s ago 26m 56.3M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (26m) 24s ago 26m 53.3M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (25m) 24s ago 25m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:07:06.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (25m) 24s ago 25m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:07:06.151 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 24s ago 24m 60.8M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:07:06.151 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (23m) 100s ago 23m 79.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:07:06.151 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (23m) 24s ago 23m 77.5M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:07:06.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:05 smithi027 ceph-mon[25136]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:06.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:05 smithi027 ceph-mon[29814]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:06.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:05 smithi120 ceph-mon[67716]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:06.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-06-21T01:07:08.108 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:07 smithi027 ceph-mon[25136]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:08.108 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:07 smithi027 ceph-mon[25136]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:08.108 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:07 smithi027 ceph-mon[29814]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:08.108 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:07 smithi027 ceph-mon[29814]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:07 smithi120 ceph-mon[67716]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:08.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:07 smithi120 ceph-mon[67716]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:08.727 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:08] "GET /metrics HTTP/1.1" 200 37922 "" "Prometheus/2.43.0" 2024-06-21T01:07:09.055 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:07:09.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:07:10.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[25136]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:10.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:10.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/3587289112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:10.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[29814]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:10.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:10.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:09 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/3587289112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:09 smithi120 ceph-mon[67716]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:10.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:09 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3587289112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:11.676 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:07:11.676 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:07:11.676 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:07:11.676 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:07:11.677 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:07:11.677 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:07:11.677 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:07:11.677 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:07:11.677 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:07:12.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:11 smithi027 ceph-mon[25136]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:12.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:11 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:12.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:11 smithi027 ceph-mon[29814]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:12.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:11 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:12.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:07:12.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:11 smithi120 ceph-mon[67716]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:12.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:11 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:13.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:12 smithi027 ceph-mon[25136]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:13.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:12 smithi027 ceph-mon[29814]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:13.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:12 smithi120 ceph-mon[67716]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:13.948 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:13 smithi027 ceph-mon[25136]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:13.948 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:13 smithi027 ceph-mon[29814]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:14.358 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:07:14.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:13 smithi120 ceph-mon[67716]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:14.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:14 smithi027 ceph-mon[25136]: from='client.? 172.21.15.27:0/4266573089' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:07:14.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:14 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/4266573089' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:07:15.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-06-21T01:07:15.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:14 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/4266573089' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:07:16.221 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:15 smithi027 ceph-mon[25136]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:16.221 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:15 smithi027 ceph-mon[29814]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:15 smithi120 ceph-mon[67716]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:17 smithi027 ceph-mon[25136]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:17 smithi027 ceph-mon[25136]: from='client.25705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:17 smithi027 ceph-mon[29814]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:17 smithi027 ceph-mon[29814]: from='client.25705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:17 smithi120 ceph-mon[67716]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:17 smithi120 ceph-mon[67716]: from='client.25705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:18.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:18] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-06-21T01:07:19.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:18 smithi027 ceph-mon[25136]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:19.182 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:18 smithi027 ceph-mon[29814]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:19.211 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:19.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:18 smithi120 ceph-mon[67716]: from='client.25711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "mon", "hosts": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:19 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:20 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.010 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:20 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:22.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:19 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:20 smithi027 ceph-mon[25136]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:20 smithi027 ceph-mon[25136]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:20 smithi027 ceph-mon[29814]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:20 smithi027 ceph-mon[29814]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:07:22.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:22.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:22.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:22.167 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:21 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-21T01:07:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:21 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-06-21T01:07:23.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:07:23.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:07:23.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: Upgrade: Setting container_image for all mgr 2024-06-21T01:07:23.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: Upgrade: It appears safe to stop mon.a 2024-06-21T01:07:23.095 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[25136]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: Upgrade: Setting container_image for all mgr 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: Upgrade: It appears safe to stop mon.a 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:23.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:22 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: Upgrade: It appears safe to stop mon.a 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:23.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:22 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:24.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[25136]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:24.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[25136]: Upgrade: Updating mon.a 2024-06-21T01:07:24.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[25136]: Deploying daemon mon.a on smithi027 2024-06-21T01:07:24.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[29814]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:24.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[29814]: Upgrade: Updating mon.a 2024-06-21T01:07:24.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:23 smithi027 ceph-mon[29814]: Deploying daemon mon.a on smithi027 2024-06-21T01:07:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:23 smithi120 ceph-mon[67716]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:23 smithi120 ceph-mon[67716]: Upgrade: Updating mon.a 2024-06-21T01:07:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:23 smithi120 ceph-mon[67716]: Deploying daemon mon.a on smithi027 2024-06-21T01:07:25.292 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:07:25.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:25 smithi027 systemd[1]: Stopping Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 2m ago 24m 17.2M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (103s) 44s ago 24m 79.1M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (2m) 2m ago 23m 50.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (3m) 44s ago 28m 440M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (13m) 2m ago 30m 497M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (30m) 2m ago 30m 77.4M 2048M 17.2.0 e1d6a67b021e 0b8357112190 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (49s) 44s ago 29m 18.6M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (29m) 2m ago 29m 60.4M 2048M 17.2.0 e1d6a67b021e 05d2411b7695 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 2m ago 24m 8120k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:07:26.030 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (2m) 44s ago 24m 15.5M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (28m) 2m ago 28m 54.6M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (28m) 2m ago 28m 58.7M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (27m) 2m ago 27m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (27m) 2m ago 27m 54.2M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (26m) 44s ago 26m 56.3M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (26m) 44s ago 26m 53.3M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (26m) 44s ago 26m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (25m) 44s ago 25m 55.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 44s ago 25m 60.8M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (23m) 2m ago 23m 79.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:07:26.031 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (23m) 44s ago 23m 77.5M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:07:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:25 smithi027 ceph-mon[29814]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:26.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE Bus STOPPING 2024-06-21T01:07:26.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T01:07:26.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE Bus STOPPED 2024-06-21T01:07:26.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE Bus STARTING 2024-06-21T01:07:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:25 smithi027 ceph-mon[25136]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:25 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a[25132]: 2024-06-21T01:07:25.995+0000 7fb79054b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:07:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:25 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a[25132]: 2024-06-21T01:07:25.995+0000 7fb79054b700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-06-21T01:07:26.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 podman[84705]: 2024-06-21 01:07:26.046798513 +0000 UTC m=+0.185701344 container died 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870) 2024-06-21T01:07:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:25 smithi120 ceph-mon[67716]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:26.444 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:07:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:07:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-21T01:07:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 1 2024-06-21T01:07:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:07:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:07:26.446 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:07:26.695 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 podman[84705]: 2024-06-21 01:07:26.429126126 +0000 UTC m=+0.568028956 container cleanup 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_CLEAN=True) 2024-06-21T01:07:26.695 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 bash[84705]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a 2024-06-21T01:07:26.695 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE Serving on http://:::9283 2024-06-21T01:07:26.695 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:26] ENGINE Bus STARTED 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi027", 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/2 daemons upgraded", 2024-06-21T01:07:26.820 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-06-21T01:07:26.821 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:07:26.821 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:07:26.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 podman[84720]: 2024-06-21 01:07:26.697359556 +0000 UTC m=+0.647454473 container remove 0b8357112190c6e0def7debf49f42d8b317a8eb85fddac2add2b12774e54b7e1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-21T01:07:26.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service: Deactivated successfully. 2024-06-21T01:07:26.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 systemd[1]: Stopped Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:07:26.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:26 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service: Consumed 28.110s CPU time. 2024-06-21T01:07:27.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 systemd[1]: Starting Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:07:27.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 podman[84907]: 2024-06-21 01:07:27.315045688 +0000 UTC m=+0.112247024 container create 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True) 2024-06-21T01:07:27.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 podman[84907]: 2024-06-21 01:07:27.22093726 +0000 UTC m=+0.018138597 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:07:27.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 podman[84907]: 2024-06-21 01:07:27.559575525 +0000 UTC m=+0.356776863 container init 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0) 2024-06-21T01:07:27.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 podman[84907]: 2024-06-21 01:07:27.563096591 +0000 UTC m=+0.360297928 container start 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: set uid:gid to 167:167 (ceph:ceph) 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc), process ceph-mon, pid 2 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: pidfile_write: ignore empty --pid-file 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: load: jerasure load: lrc 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: RocksDB version: 7.9.2 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Git sha 0 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Compile date 2024-06-19 15:40:36 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: DB SUMMARY 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: DB Session ID: 2IG74BQW5HVL6TQ7EMTZ 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: CURRENT file: CURRENT 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: IDENTITY file: IDENTITY 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2888 Bytes 2024-06-21T01:07:27.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 2246341 ; 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.error_if_exists: 0 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.create_if_missing: 0 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.paranoid_checks: 1 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.env: 0x55b2035d7740 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.fs: PosixFileSystem 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.info_log: 0x55b20415b0c0 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_file_opening_threads: 16 2024-06-21T01:07:27.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.statistics: (nil) 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.use_fsync: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_log_file_size: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.keep_log_file_num: 1000 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.recycle_log_file_num: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_fallocate: 1 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_mmap_reads: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_mmap_writes: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.use_direct_reads: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.create_missing_column_families: 0 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.db_log_dir: 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.wal_dir: 2024-06-21T01:07:27.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.advise_random_on_open: 1 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.db_write_buffer_size: 0 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.write_buffer_manager: 0x55b20416ac80 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.rate_limiter: (nil) 2024-06-21T01:07:27.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.wal_recovery_mode: 2 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enable_thread_tracking: 0 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enable_pipelined_write: 0 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.unordered_write: 0 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.row_cache: None 2024-06-21T01:07:27.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.wal_filter: None 2024-06-21T01:07:27.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-21T01:07:27.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_ingest_behind: 0 2024-06-21T01:07:27.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.two_write_queues: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.manual_wal_flush: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.wal_compression: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.atomic_flush: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.log_readahead_size: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.best_efforts_recovery: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.allow_data_in_errors: 0 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.db_host_id: __hostname__ 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-21T01:07:27.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_background_jobs: 2 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_background_compactions: -1 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_subcompactions: 1 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_total_wal_size: 0 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_open_files: -1 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bytes_per_sync: 0 2024-06-21T01:07:27.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_readahead_size: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_background_flushes: -1 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Compression algorithms supported: 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kZSTD supported: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kXpressCompression supported: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kBZip2Compression supported: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kLZ4Compression supported: 1 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kZlibCompression supported: 1 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kLZ4HCCompression supported: 1 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: kSnappyCompression supported: 1 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-21T01:07:27.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.merge_operator: 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_filter: None 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_filter_factory: None 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.sst_partitioner_factory: None 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-21T01:07:27.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b20415ad20) 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_type: 0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_index_type: 0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_shortening: 1 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: checksum: 4 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: no_block_cache: 0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache: 0x55b2041531f0 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_options: 2024-06-21T01:07:27.989 INFO:journalctl@ceph.mon.a.smithi027.stdout: capacity : 536870912 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_shard_bits : 4 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: strict_capacity_limit : 0 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_compressed: (nil) 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: persistent_cache: (nil) 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size: 4096 2024-06-21T01:07:27.990 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size_deviation: 10 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_restart_interval: 16 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_block_restart_interval: 1 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_block_size: 4096 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: partition_filters: 0 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: use_delta_encoding: 1 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: filter_policy: bloomfilter 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: whole_key_filtering: 1 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: verify_compression: 0 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: format_version: 5 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_index_compression: 1 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_align: 0 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_auto_readahead_size: 262144 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: prepopulate_block_cache: 0 2024-06-21T01:07:27.991 INFO:journalctl@ceph.mon.a.smithi027.stdout: initial_auto_readahead_size: 8192 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.write_buffer_size: 33554432 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_write_buffer_number: 2 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression: NoCompression 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression: Disabled 2024-06-21T01:07:27.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.prefix_extractor: nullptr 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.num_levels: 7 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-21T01:07:27.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-21T01:07:27.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.level: 32767 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.strategy: 0 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.enabled: false 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:07:27.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.target_file_size_base: 67108864 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-21T01:07:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.arena_block_size: 1048576 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-21T01:07:27.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.disable_auto_compactions: 0 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-21T01:07:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.table_properties_collectors: 2024-06-21T01:07:27.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.inplace_update_support: 0 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.bloom_locality: 0 2024-06-21T01:07:28.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.max_successive_merges: 0 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.paranoid_file_checks: 0 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.force_consistency_checks: 1 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.report_bg_io_stats: 0 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.ttl: 2592000 2024-06-21T01:07:28.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-21T01:07:28.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-21T01:07:28.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-21T01:07:28.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enable_blob_files: false 2024-06-21T01:07:28.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.min_blob_size: 0 2024-06-21T01:07:28.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_file_size: 268435456 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-21T01:07:28.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.blob_file_starting_level: 0 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26878, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-06-21T01:07:28.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 72f2d492-df0d-43bd-a951-2439928e1386 2024-06-21T01:07:28.005 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932047590439, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-06-21T01:07:28.005 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-06-21T01:07:28.005 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932047598226, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 2075141, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26872, "largest_seqno": 28958, "table_properties": {"data_size": 2067144, "index_size": 4741, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2245, "raw_key_size": 22083, "raw_average_key_size": 24, "raw_value_size": 2048835, "raw_average_value_size": 2296, "num_data_blocks": 213, "num_entries": 892, "num_filter_entries": 892, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718932047, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "72f2d492-df0d-43bd-a951-2439928e1386", "db_session_id": "2IG74BQW5HVL6TQ7EMTZ", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-06-21T01:07:28.005 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932047598350, "job": 1, "event": "recovery_finished"} 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b20417ee00 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: DB pointer 0x55b204274000 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-21T01:07:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** DB Stats ** 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 1/0 1.98 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: L6 1/0 9.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 11.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:07:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 517.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-21T01:07:28.009 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T01:07:28.010 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 161.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:07:28.011 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 161.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:07:28.011 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T01:07:28.011 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache BinnedLRUCache@0x55b2041531f0#2 capacity: 512.00 MB usage: 7.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-06-21T01:07:28.011 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-06-21T01:07:28.012 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.012 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T01:07:28.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:28.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???) e3 preinit fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:28.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).mds e1 new map 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).mds e1 print_map 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: e1 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: legacy client fscid: -1 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout: No filesystems configured 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-21T01:07:28.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:28.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:28.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:28.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 ceph-mon[84934]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-06-21T01:07:28.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 bash[84907]: 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 2024-06-21T01:07:28.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:27 smithi027 systemd[1]: Started Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:07:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:28] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-06-21T01:07:29.338 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: ignoring --setuser ceph since I am not root 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: ignoring --setgroup ceph since I am not root 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mgr[60759]: -- 172.21.15.120:0/3381033748 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d3afe470e0 con 0x55d3afe24800 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:29.191+0000 7fef35e14040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.339 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2929542271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='client.25741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: mon.a calling monitor election 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: monmap epoch 3 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: last_changed 2024-06-21T00:38:22.715626+0000 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: min_mon_release 17 (quincy) 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: election_strategy: 1 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:29.340 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: fsmap 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: overall HEALTH_OK 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: from='mgr.15372 ' entity='' 2024-06-21T01:07:29.341 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-mon[67716]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-21T01:07:29.352 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ignoring --setuser ceph since I am not root 2024-06-21T01:07:29.352 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ignoring --setgroup ceph since I am not root 2024-06-21T01:07:29.352 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mgr[63256]: -- 172.21.15.27:0/4100247289 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55586bd870e0 con 0x55586bd64800 2024-06-21T01:07:29.352 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:29.195+0000 7f3fa85bd040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-06-21T01:07:29.352 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:29.354+0000 7f3fa85bd040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2929542271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='client.25741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: mon.a calling monitor election 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: monmap epoch 3 2024-06-21T01:07:29.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: last_changed 2024-06-21T00:38:22.715626+0000 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: min_mon_release 17 (quincy) 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: election_strategy: 1 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:29.354 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: fsmap 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: overall HEALTH_OK 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: from='mgr.15372 ' entity='' 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[84934]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='client.25723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.355 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='client.? 172.21.15.27:0/2929542271' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='client.25741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='mgr.15372 172.21.15.27:0/4208477952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: mon.a calling monitor election 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: monmap epoch 3 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: last_changed 2024-06-21T00:38:22.715626+0000 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: min_mon_release 17 (quincy) 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: election_strategy: 1 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:29.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: fsmap 2024-06-21T01:07:29.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: osdmap e112: 8 total, 8 up, 8 in 2024-06-21T01:07:29.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: mgrmap e41: y(active, since 3m), standbys: x 2024-06-21T01:07:29.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: overall HEALTH_OK 2024-06-21T01:07:29.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: from='mgr.15372 ' entity='' 2024-06-21T01:07:29.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-mon[29814]: mgrmap e42: y(active, since 3m), standbys: x 2024-06-21T01:07:29.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:29.340+0000 7fef35e14040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:07:29.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:29.411+0000 7fef35e14040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T01:07:29.688 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:29.428+0000 7f3fa85bd040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-06-21T01:07:29.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:29.689+0000 7f3fa85bd040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T01:07:29.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:29.754+0000 7f3fa85bd040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T01:07:30.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:29.654+0000 7fef35e14040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-06-21T01:07:30.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:29.715+0000 7fef35e14040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-06-21T01:07:31.030 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:30.758+0000 7f3fa85bd040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T01:07:31.031 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:30.882+0000 7f3fa85bd040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T01:07:31.031 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:30.944+0000 7f3fa85bd040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T01:07:31.031 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:30.649+0000 7fef35e14040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-06-21T01:07:31.031 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:30.765+0000 7fef35e14040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-06-21T01:07:31.031 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:30.823+0000 7fef35e14040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-06-21T01:07:31.313 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-mon[67716]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-21T01:07:31.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-mon[84934]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-21T01:07:31.315 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:31.037+0000 7fef35e14040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T01:07:31.315 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:31.100+0000 7fef35e14040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T01:07:31.315 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:31.167+0000 7fef35e14040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T01:07:31.315 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:31.314+0000 7fef35e14040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:07:31.316 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-mon[29814]: mgrmap e43: y(active, since 3m), standbys: x 2024-06-21T01:07:31.316 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:31.176+0000 7f3fa85bd040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-06-21T01:07:31.316 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:31.241+0000 7f3fa85bd040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-06-21T01:07:31.317 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:31.313+0000 7f3fa85bd040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-06-21T01:07:31.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:31.471+0000 7f3fa85bd040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-06-21T01:07:32.058 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:31.739+0000 7fef35e14040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T01:07:32.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:31.928+0000 7f3fa85bd040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-06-21T01:07:32.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.060+0000 7fef35e14040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T01:07:32.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.128+0000 7fef35e14040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T01:07:32.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.279+0000 7fef35e14040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T01:07:32.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:32.270+0000 7f3fa85bd040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-06-21T01:07:32.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:32.342+0000 7f3fa85bd040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-06-21T01:07:32.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:32.504+0000 7f3fa85bd040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-06-21T01:07:32.847 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.596+0000 7fef35e14040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T01:07:32.848 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.663+0000 7fef35e14040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T01:07:33.111 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:32.846+0000 7f3fa85bd040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-06-21T01:07:33.111 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:32.912+0000 7f3fa85bd040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-06-21T01:07:33.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.849+0000 7fef35e14040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T01:07:33.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:32.998+0000 7fef35e14040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T01:07:33.116 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:33.062+0000 7fef35e14040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T01:07:33.429 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:33.112+0000 7f3fa85bd040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-06-21T01:07:33.429 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:33.271+0000 7f3fa85bd040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-06-21T01:07:33.429 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:33.340+0000 7f3fa85bd040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-06-21T01:07:33.429 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:33.430+0000 7f3fa85bd040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T01:07:33.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:33.148+0000 7fef35e14040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-06-21T01:07:33.616 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:33.208+0000 7fef35e14040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T01:07:33.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:33.495+0000 7f3fa85bd040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: Standby manager daemon x restarted 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: Standby manager daemon x started 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-mon[67716]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:33.753+0000 7fef35e14040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:33.914+0000 7fef35e14040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T01:07:34.019 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:07:33] ENGINE Bus STARTING 2024-06-21T01:07:34.020 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: CherryPy Checker: 2024-06-21T01:07:34.020 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: The Application mounted at '' has an empty config. 2024-06-21T01:07:34.020 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: 2024-06-21T01:07:34.020 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:07:34] ENGINE Serving on http://:::9283 2024-06-21T01:07:34.020 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x[60755]: [21/Jun/2024:01:07:34] ENGINE Bus STARTED 2024-06-21T01:07:34.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:34.083+0000 7f3fa85bd040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-06-21T01:07:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: Standby manager daemon x restarted 2024-06-21T01:07:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: Standby manager daemon x started 2024-06-21T01:07:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:07:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:07:34.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:07:34.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[84934]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:07:34.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: Standby manager daemon x restarted 2024-06-21T01:07:34.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: Standby manager daemon x started 2024-06-21T01:07:34.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-06-21T01:07:34.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-21T01:07:34.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-06-21T01:07:34.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:33 smithi027 ceph-mon[29814]: from='mgr.? 172.21.15.120:0/425550165' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:34.256+0000 7f3fa85bd040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:34] ENGINE Bus STARTING 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: CherryPy Checker: 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: The Application mounted at '' has an empty config. 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:34] ENGINE Serving on http://:::9283 2024-06-21T01:07:34.547 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:34] ENGINE Bus STARTED 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: mgrmap e44: y(active, since 3m), standbys: x 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: Active manager daemon y restarted 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: Activating manager daemon y 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: mgrmap e45: y(active, starting, since 0.00637067s), standbys: x 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:07:35.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: Manager daemon y is now available 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:35.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: mgrmap e44: y(active, since 3m), standbys: x 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: Active manager daemon y restarted 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: Activating manager daemon y 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: mgrmap e45: y(active, starting, since 0.00637067s), standbys: x 2024-06-21T01:07:35.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:07:35.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: Manager daemon y is now available 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:34 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:35.272 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:35.268+0000 7f3f47d87640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-06-21T01:07:35.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: mgrmap e44: y(active, since 3m), standbys: x 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: Active manager daemon y restarted 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: Activating manager daemon y 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: mgrmap e45: y(active, starting, since 0.00637067s), standbys: x 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:07:35.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: Manager daemon y is now available 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:35.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:34 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-06-21T01:07:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:36 smithi120 ceph-mon[67716]: mgrmap e46: y(active, since 1.00888s), standbys: x 2024-06-21T01:07:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:36 smithi120 ceph-mon[67716]: [21/Jun/2024:01:07:35] ENGINE Bus STARTING 2024-06-21T01:07:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:36 smithi120 ceph-mon[67716]: [21/Jun/2024:01:07:36] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:07:36.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[29814]: mgrmap e46: y(active, since 1.00888s), standbys: x 2024-06-21T01:07:36.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[29814]: [21/Jun/2024:01:07:35] ENGINE Bus STARTING 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[29814]: [21/Jun/2024:01:07:36] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[84934]: mgrmap e46: y(active, since 1.00888s), standbys: x 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[84934]: [21/Jun/2024:01:07:35] ENGINE Bus STARTING 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-mon[84934]: [21/Jun/2024:01:07:36] ENGINE Serving on http://172.21.15.27:8765 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:36.143+0000 7f3f000f0640 -1 log_channel(cephadm) log [ERR] : [21/Jun/2024:01:07:36] ENGINE Error in HTTPServer.serve 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: Traceback (most recent call last): 2024-06-21T01:07:36.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._connections.run(self.expiration_interval) 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._run(expiration_interval) 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: s = self.context.wrap_socket( 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: return self.sslsocket_class._create( 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self.do_handshake() 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:07:36.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: self._sslobj.do_handshake() 2024-06-21T01:07:36.481 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:07:36.481 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:37 smithi120 ceph-mon[67716]: [21/Jun/2024:01:07:36] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:37 smithi120 ceph-mon[67716]: [21/Jun/2024:01:07:36] ENGINE Bus STARTED 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:37 smithi120 ceph-mon[67716]: [21/Jun/2024:01:07:36] ENGINE Error in HTTPServer.serve 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout: Traceback (most recent call last): 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:07:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._run(expiration_interval) 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: s = self.context.wrap_socket( 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: return self.sslsocket_class._create( 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: self.do_handshake() 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: self._sslobj.do_handshake() 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:07:37.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:37 smithi120 ceph-mon[67716]: mgrmap e47: y(active, since 2s), standbys: x 2024-06-21T01:07:37.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:37 smithi120 ceph-mon[67716]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:37.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[29814]: [21/Jun/2024:01:07:36] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:07:37.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[29814]: [21/Jun/2024:01:07:36] ENGINE Bus STARTED 2024-06-21T01:07:37.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[29814]: [21/Jun/2024:01:07:36] ENGINE Error in HTTPServer.serve 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:07:37.671 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[29814]: mgrmap e47: y(active, since 2s), standbys: x 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[29814]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[84934]: [21/Jun/2024:01:07:36] ENGINE Serving on https://172.21.15.27:7150 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[84934]: [21/Jun/2024:01:07:36] ENGINE Bus STARTED 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[84934]: [21/Jun/2024:01:07:36] ENGINE Error in HTTPServer.serve 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-21T01:07:37.672 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-06-21T01:07:37.673 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-21T01:07:37.674 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-06-21T01:07:37.674 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-06-21T01:07:37.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[84934]: mgrmap e47: y(active, since 2s), standbys: x 2024-06-21T01:07:37.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:37 smithi027 ceph-mon[84934]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:38] "GET /metrics HTTP/1.1" 200 35125 "" "Prometheus/2.43.0" 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: mgrmap e48: y(active, since 4s), standbys: x 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:39 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: mgrmap e48: y(active, since 4s), standbys: x 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: mgrmap e48: y(active, since 4s), standbys: x 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:39 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:40 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.824 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:40 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.824 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:40 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.824 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:40 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:40.824 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:40 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:41.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:41 smithi120 ceph-mon[67716]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:41 smithi027 ceph-mon[84934]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:41 smithi027 ceph-mon[29814]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: Detected new or changed devices on smithi027 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:07:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:07:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:43.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:43 smithi120 ceph-mon[67716]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: Detected new or changed devices on smithi027 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[84934]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: Detected new or changed devices on smithi027 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:07:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-21T01:07:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.conf 2024-06-21T01:07:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:43.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:43 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.conf 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[29814]: Upgrade: It appears safe to stop mon.c 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:44.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:44.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:44.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:44.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-21T01:07:44.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:44 smithi027 ceph-mon[84934]: Upgrade: It appears safe to stop mon.c 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: Updating smithi120:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: Updating smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: Updating smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/config/ceph.client.admin.keyring 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-06-21T01:07:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:44 smithi120 ceph-mon[67716]: Upgrade: It appears safe to stop mon.c 2024-06-21T01:07:45.912 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[84934]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[84934]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[29814]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[29814]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:45.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:45 smithi027 ceph-mon[29814]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:45 smithi120 ceph-mon[67716]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-21T01:07:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:45 smithi120 ceph-mon[67716]: from='mgr.25753 ' entity='mgr.y' 2024-06-21T01:07:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:07:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:07:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:07:46.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:46 smithi027 ceph-mon[84934]: Upgrade: Updating mon.c 2024-06-21T01:07:46.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:46 smithi027 ceph-mon[84934]: Deploying daemon mon.c on smithi027 2024-06-21T01:07:46.863 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:46 smithi027 ceph-mon[29814]: Upgrade: Updating mon.c 2024-06-21T01:07:46.863 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:46 smithi027 ceph-mon[29814]: Deploying daemon mon.c on smithi027 2024-06-21T01:07:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:46 smithi120 ceph-mon[67716]: Upgrade: Updating mon.c 2024-06-21T01:07:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:46 smithi120 ceph-mon[67716]: Deploying daemon mon.c on smithi027 2024-06-21T01:07:47.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:46 smithi027 systemd[1]: Stopping Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:07:47.463 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE Bus STOPPING 2024-06-21T01:07:47.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c[29810]: 2024-06-21T01:07:47.147+0000 7fcbac93f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:07:47.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c[29810]: 2024-06-21T01:07:47.147+0000 7fcbac93f700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-06-21T01:07:47.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 podman[90065]: 2024-06-21 01:07:47.169024672 +0000 UTC m=+0.149918556 container died 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, release=754, version=8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-21T01:07:47.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 podman[90065]: 2024-06-21 01:07:47.564604109 +0000 UTC m=+0.545497978 container cleanup 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, GIT_CLEAN=True, version=8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, ceph=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8) 2024-06-21T01:07:47.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 bash[90065]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c 2024-06-21T01:07:47.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T01:07:47.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE Bus STOPPED 2024-06-21T01:07:47.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE Bus STARTING 2024-06-21T01:07:48.012 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE Serving on http://:::9283 2024-06-21T01:07:48.012 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:47 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:07:47] ENGINE Bus STARTED 2024-06-21T01:07:48.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 podman[90077]: 2024-06-21 01:07:47.833097365 +0000 UTC m=+0.659418931 container remove 05d2411b7695230d28e3cfb404b02dbe08f166a6514cf4cea497b30932b76269 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream) 2024-06-21T01:07:48.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.c.service: Deactivated successfully. 2024-06-21T01:07:48.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 systemd[1]: Stopped Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:07:48.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:47 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.c.service: Consumed 16.218s CPU time. 2024-06-21T01:07:48.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 systemd[1]: Starting Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:07:48.682 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:48] "GET /metrics HTTP/1.1" 200 35125 "" "Prometheus/2.43.0" 2024-06-21T01:07:48.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 podman[90217]: 2024-06-21 01:07:48.333531067 +0000 UTC m=+0.017789843 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:07:48.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 podman[90217]: 2024-06-21 01:07:48.442333946 +0000 UTC m=+0.126592719 container create 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:07:48.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 podman[90217]: 2024-06-21 01:07:48.645156865 +0000 UTC m=+0.329415639 container init 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:07:48.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 podman[90217]: 2024-06-21 01:07:48.648519447 +0000 UTC m=+0.332778220 container start 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-06-21T01:07:48.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: set uid:gid to 167:167 (ceph:ceph) 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc), process ceph-mon, pid 2 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: pidfile_write: ignore empty --pid-file 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: load: jerasure load: lrc 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: RocksDB version: 7.9.2 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Git sha 0 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Compile date 2024-06-19 15:40:36 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: DB SUMMARY 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: DB Session ID: Z2X91TAMQRX6BKCYWYF2 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: CURRENT file: CURRENT 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: IDENTITY file: IDENTITY 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3197 Bytes 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-06-21T01:07:48.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 843505 ; 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.error_if_exists: 0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.create_if_missing: 0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.paranoid_checks: 1 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.env: 0x559a45951740 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.fs: PosixFileSystem 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.info_log: 0x559a47d330c0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_file_opening_threads: 16 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.statistics: (nil) 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.use_fsync: 0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_log_file_size: 0 2024-06-21T01:07:48.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.keep_log_file_num: 1000 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.recycle_log_file_num: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_fallocate: 1 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_mmap_reads: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_mmap_writes: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.use_direct_reads: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.create_missing_column_families: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.db_log_dir: 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.wal_dir: 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-21T01:07:48.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.advise_random_on_open: 1 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.db_write_buffer_size: 0 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.write_buffer_manager: 0x559a47d42c80 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.rate_limiter: (nil) 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.wal_recovery_mode: 2 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enable_thread_tracking: 0 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enable_pipelined_write: 0 2024-06-21T01:07:48.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.unordered_write: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.row_cache: None 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.wal_filter: None 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_ingest_behind: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.two_write_queues: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.manual_wal_flush: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.wal_compression: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.atomic_flush: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-21T01:07:48.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.log_readahead_size: 0 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.best_efforts_recovery: 0 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.allow_data_in_errors: 0 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.db_host_id: __hostname__ 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_background_jobs: 2 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_background_compactions: -1 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_subcompactions: 1 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-21T01:07:48.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_total_wal_size: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_open_files: -1 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bytes_per_sync: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_readahead_size: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_background_flushes: -1 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Compression algorithms supported: 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kZSTD supported: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kXpressCompression supported: 0 2024-06-21T01:07:48.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kBZip2Compression supported: 0 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kLZ4Compression supported: 1 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kZlibCompression supported: 1 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kLZ4HCCompression supported: 1 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: kSnappyCompression supported: 1 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.merge_operator: 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_filter: None 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_filter_factory: None 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.sst_partitioner_factory: None 2024-06-21T01:07:48.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559a47d32d20) 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_type: 0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_index_type: 0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_shortening: 1 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: checksum: 4 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: no_block_cache: 0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache: 0x559a47d2b1f0 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-21T01:07:48.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_options: 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: capacity : 536870912 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_shard_bits : 4 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: strict_capacity_limit : 0 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_compressed: (nil) 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: persistent_cache: (nil) 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size: 4096 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size_deviation: 10 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_restart_interval: 16 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_block_restart_interval: 1 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_block_size: 4096 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: partition_filters: 0 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: use_delta_encoding: 1 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: filter_policy: bloomfilter 2024-06-21T01:07:48.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: whole_key_filtering: 1 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: verify_compression: 0 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: format_version: 5 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_index_compression: 1 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_align: 0 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_auto_readahead_size: 262144 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: prepopulate_block_cache: 0 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: initial_auto_readahead_size: 8192 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.write_buffer_size: 33554432 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_write_buffer_number: 2 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression: NoCompression 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression: Disabled 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.prefix_extractor: nullptr 2024-06-21T01:07:48.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.num_levels: 7 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-21T01:07:48.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.level: 32767 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.strategy: 0 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.enabled: false 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.target_file_size_base: 67108864 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-21T01:07:48.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.arena_block_size: 1048576 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-21T01:07:48.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.disable_auto_compactions: 0 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.table_properties_collectors: 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.inplace_update_support: 0 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-21T01:07:48.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.bloom_locality: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.max_successive_merges: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.paranoid_file_checks: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.force_consistency_checks: 1 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.report_bg_io_stats: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.ttl: 2592000 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enable_blob_files: false 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.min_blob_size: 0 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_file_size: 268435456 2024-06-21T01:07:48.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.blob_file_starting_level: 0 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32139, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 355edfb9-f552-4bee-8a82-9776fe3c64fc 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932068675724, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932068678347, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 482137, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32144, "largest_seqno": 32422, "table_properties": {"data_size": 479838, "index_size": 967, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2920, "raw_average_key_size": 23, "raw_value_size": 476897, "raw_average_value_size": 3908, "num_data_blocks": 43, "num_entries": 122, "num_filter_entries": 122, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718932068, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "355edfb9-f552-4bee-8a82-9776fe3c64fc", "db_session_id": "Z2X91TAMQRX6BKCYWYF2", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-06-21T01:07:48.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718932068678449, "job": 1, "event": "recovery_finished"} 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559a47d56e00 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: DB pointer 0x559a47e4c000 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-21T01:07:48.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** DB Stats ** 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-21T01:07:48.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 470.84 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 332.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: L6 1/0 11.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 2/0 11.54 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 332.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 332.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 332.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 63.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 63.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache BinnedLRUCache@0x559a47d2b1f0#2 capacity: 512.00 MB usage: 1.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-21T01:07:48.705 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: starting mon.c rank 1 at public addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] at bind addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???) e3 preinit fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).mds e1 new map 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).mds e1 print_map 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: e1 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: legacy client fscid: -1 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: No filesystems configured 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-06-21T01:07:48.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 ceph-mon[90235]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-06-21T01:07:48.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 bash[90217]: 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 2024-06-21T01:07:48.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:48 smithi027 systemd[1]: Started Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[90235]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[84934]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:07:49.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:49.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:49.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:49 smithi120 ceph-mon[67716]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-06-21T01:07:49.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:49.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:49.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:53.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:53 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:07:53.691+0000 7f3f47d87640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-06-21T01:07:54.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:54.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.a calling monitor election 2024-06-21T01:07:54.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.c calling monitor election 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.b calling monitor election 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: overall HEALTH_OK 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.c calling monitor election 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.a calling monitor election 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: monmap epoch 4 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: last_changed 2024-06-21T01:07:49.154642+0000 2024-06-21T01:07:54.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: min_mon_release 19 (squid) 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: election_strategy: 1 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: fsmap 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: mgrmap e48: y(active, since 19s), standbys: x 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: overall HEALTH_OK 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.618 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:54.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.a calling monitor election 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.c calling monitor election 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.b calling monitor election 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: overall HEALTH_OK 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.c calling monitor election 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.a calling monitor election 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:54.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: monmap epoch 4 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: last_changed 2024-06-21T01:07:49.154642+0000 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: min_mon_release 19 (squid) 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: election_strategy: 1 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: fsmap 2024-06-21T01:07:54.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: mgrmap e48: y(active, since 19s), standbys: x 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: overall HEALTH_OK 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.a calling monitor election 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.c calling monitor election 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.b calling monitor election 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-06-21T01:07:54.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: overall HEALTH_OK 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.c calling monitor election 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.a calling monitor election 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: monmap epoch 4 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: fsid 47651940-2f66-11ef-bc9d-c7b262605968 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: last_changed 2024-06-21T01:07:49.154642+0000 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: created 2024-06-21T00:36:48.093788+0000 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: min_mon_release 19 (squid) 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: election_strategy: 1 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-06-21T01:07:54.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: 1: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: 2: [v2:172.21.15.120:3300/0,v1:172.21.15.120:6789/0] mon.b 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: fsmap 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: osdmap e113: 8 total, 8 up, 8 in 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: mgrmap e48: y(active, since 19s), standbys: x 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: overall HEALTH_OK 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:54.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:07:55.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:55 smithi120 ceph-mon[67716]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:55.665 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:55 smithi027 ceph-mon[84934]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:55.665 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:55 smithi027 ceph-mon[90235]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:57.207 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:07:57.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:57 smithi027 ceph-mon[90235]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:57.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:57 smithi027 ceph-mon[84934]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:57.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:57 smithi120 ceph-mon[67716]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:07:57.907 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:07:57.907 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 18s ago 24m 19.5M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (2m) 19s ago 24m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (3m) 18s ago 23m 53.7M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (3m) 19s ago 29m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (13m) 18s ago 31m 472M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (30s) 18s ago 31m 31.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (80s) 19s ago 29m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 starting - - - 2048M 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 18s ago 25m 8594k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (3m) 19s ago 25m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (29m) 18s ago 29m 54.8M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (28m) 18s ago 28m 58.8M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (28m) 18s ago 28m 49.4M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (27m) 18s ago 27m 54.1M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:07:57.908 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (27m) 19s ago 27m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (27m) 19s ago 27m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (26m) 19s ago 26m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (26m) 19s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (2m) 19s ago 25m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (24m) 18s ago 24m 80.3M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:07:57.909 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (23m) 19s ago 23m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:07:58.361 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:07:58.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:07:58.363 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:07:58.363 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-21T01:07:58.363 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 5 2024-06-21T01:07:58.363 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:07:58.363 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:07:58.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[84934]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[84934]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[84934]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[90235]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[90235]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-mon[90235]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:58 smithi120 ceph-mon[67716]: from='client.54138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:58 smithi120 ceph-mon[67716]: from='client.54144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:58 smithi120 ceph-mon[67716]: from='client.54150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi027", 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:07:58.724 INFO:teuthology.orchestra.run.smithi027.stdout: "mon" 2024-06-21T01:07:58.725 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:07:58.725 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-06-21T01:07:58.725 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-06-21T01:07:58.725 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:07:58.725 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:07:58.725 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:07:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:07:58] "GET /metrics HTTP/1.1" 200 37938 "" "Prometheus/2.43.0" 2024-06-21T01:07:59.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:59.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3201323516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:59.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:59.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:07:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.629 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:59.629 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3201323516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3201323516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:07:59.630 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:07:59.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='client.54162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:07:59.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:07:59.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:07:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:01.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:01 smithi120 ceph-mon[67716]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:01.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:01 smithi027 ceph-mon[84934]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:01.617 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:01 smithi027 ceph-mon[90235]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:03.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: Reconfiguring mon.a (monmap changed)... 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[84934]: Reconfiguring daemon mon.a on smithi027 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: Reconfiguring mon.a (monmap changed)... 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:03 smithi027 ceph-mon[90235]: Reconfiguring daemon mon.a on smithi027 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:03.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:03.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: Reconfiguring mon.a (monmap changed)... 2024-06-21T01:08:03.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:03.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:03.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:03.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:03 smithi120 ceph-mon[67716]: Reconfiguring daemon mon.a on smithi027 2024-06-21T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: Reconfiguring mgr.y (monmap changed)... 2024-06-21T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T01:08:05.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:05.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: Reconfiguring mgr.y (monmap changed)... 2024-06-21T01:08:05.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: Reconfiguring mgr.y (monmap changed)... 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: Reconfiguring daemon mgr.y on smithi027 2024-06-21T01:08:05.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:05.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:05.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:06.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: Reconfiguring mon.c (monmap changed)... 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: Reconfiguring daemon mon.c on smithi027 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: Reconfiguring mon.c (monmap changed)... 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: Reconfiguring daemon mon.c on smithi027 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: Reconfiguring mon.c (monmap changed)... 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: Reconfiguring daemon mon.c on smithi027 2024-06-21T01:08:07.119 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.120 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:07.120 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:07.120 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[84934]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[84934]: Reconfiguring osd.0 (monmap changed)... 2024-06-21T01:08:07.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.0 on smithi027 2024-06-21T01:08:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[90235]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[90235]: Reconfiguring osd.0 (monmap changed)... 2024-06-21T01:08:07.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:07 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.0 on smithi027 2024-06-21T01:08:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:07 smithi120 ceph-mon[67716]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:07 smithi120 ceph-mon[67716]: Reconfiguring osd.0 (monmap changed)... 2024-06-21T01:08:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:07 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.0 on smithi027 2024-06-21T01:08:08.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:08] "GET /metrics HTTP/1.1" 200 37937 "" "Prometheus/2.43.0" 2024-06-21T01:08:09.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: Reconfiguring osd.1 (monmap changed)... 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.1 on smithi027 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: Reconfiguring osd.1 (monmap changed)... 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:08:09.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:09.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.1 on smithi027 2024-06-21T01:08:09.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:09.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: Reconfiguring osd.1 (monmap changed)... 2024-06-21T01:08:09.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:08:09.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:09.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.1 on smithi027 2024-06-21T01:08:09.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:09.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:11.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: Reconfiguring osd.2 (monmap changed)... 2024-06-21T01:08:11.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:08:11.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.2 on smithi027 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: Reconfiguring osd.2 (monmap changed)... 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:11.082 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:10 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.2 on smithi027 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: Reconfiguring osd.2 (monmap changed)... 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:11.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:10 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.2 on smithi027 2024-06-21T01:08:11.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[90235]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:11.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[84934]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:08:11.827 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:11 smithi120 ceph-mon[67716]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:08:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:12.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: Reconfiguring osd.3 (monmap changed)... 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.3 on smithi027 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: Reconfiguring osd.3 (monmap changed)... 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.3 on smithi027 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:12.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: Reconfiguring osd.3 (monmap changed)... 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.3 on smithi027 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:14.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[84934]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:14.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[84934]: Reconfiguring rgw.foo.smithi027.dsjkpz (monmap changed)... 2024-06-21T01:08:14.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[84934]: Reconfiguring daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:08:14.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[90235]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:14.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[90235]: Reconfiguring rgw.foo.smithi027.dsjkpz (monmap changed)... 2024-06-21T01:08:14.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:13 smithi027 ceph-mon[90235]: Reconfiguring daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:08:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:13 smithi120 ceph-mon[67716]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:13 smithi120 ceph-mon[67716]: Reconfiguring rgw.foo.smithi027.dsjkpz (monmap changed)... 2024-06-21T01:08:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:13 smithi120 ceph-mon[67716]: Reconfiguring daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:08:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: Reconfiguring mon.b (monmap changed)... 2024-06-21T01:08:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[84934]: Reconfiguring daemon mon.b on smithi120 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: Reconfiguring mon.b (monmap changed)... 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:15.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:14 smithi027 ceph-mon[90235]: Reconfiguring daemon mon.b on smithi120 2024-06-21T01:08:15.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:15.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: Reconfiguring mon.b (monmap changed)... 2024-06-21T01:08:15.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-21T01:08:15.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-21T01:08:15.275 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:15.275 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:14 smithi120 ceph-mon[67716]: Reconfiguring daemon mon.b on smithi120 2024-06-21T01:08:16.036 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:16.036 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.036 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.036 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:16.036 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:16.037 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-21T01:08:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: Reconfiguring mgr.x (monmap changed)... 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: Reconfiguring daemon mgr.x on smithi120 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:08:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: Reconfiguring mgr.x (monmap changed)... 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: Reconfiguring daemon mgr.x on smithi120 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:08:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: Reconfiguring mgr.x (monmap changed)... 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: Reconfiguring daemon mgr.x on smithi120 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:08:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:17 smithi120 ceph-mon[67716]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:17 smithi120 ceph-mon[67716]: Reconfiguring osd.4 (monmap changed)... 2024-06-21T01:08:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:17 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.4 on smithi120 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[84934]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[84934]: Reconfiguring osd.4 (monmap changed)... 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.4 on smithi120 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[90235]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[90235]: Reconfiguring osd.4 (monmap changed)... 2024-06-21T01:08:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:17 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.4 on smithi120 2024-06-21T01:08:18.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:18] "GET /metrics HTTP/1.1" 200 37937 "" "Prometheus/2.43.0" 2024-06-21T01:08:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: Reconfiguring osd.5 (monmap changed)... 2024-06-21T01:08:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:08:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:19.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.5 on smithi120 2024-06-21T01:08:19.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:19.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: Reconfiguring osd.5 (monmap changed)... 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.5 on smithi120 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: Reconfiguring osd.5 (monmap changed)... 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.5 on smithi120 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: Reconfiguring osd.6 (monmap changed)... 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:20.496 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:20 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.6 on smithi120 2024-06-21T01:08:20.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:20.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: Reconfiguring osd.6 (monmap changed)... 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.6 on smithi120 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:20.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: Reconfiguring osd.6 (monmap changed)... 2024-06-21T01:08:20.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:08:20.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:20.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:20 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.6 on smithi120 2024-06-21T01:08:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:21 smithi120 ceph-mon[67716]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:08:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[84934]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:08:21.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[90235]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:08:21.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:22.541 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:22 smithi120 ceph-mon[67716]: Reconfiguring osd.7 (monmap changed)... 2024-06-21T01:08:22.542 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:22 smithi120 ceph-mon[67716]: Reconfiguring daemon osd.7 on smithi120 2024-06-21T01:08:22.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:22 smithi027 ceph-mon[84934]: Reconfiguring osd.7 (monmap changed)... 2024-06-21T01:08:22.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:22 smithi027 ceph-mon[84934]: Reconfiguring daemon osd.7 on smithi120 2024-06-21T01:08:22.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:22 smithi027 ceph-mon[90235]: Reconfiguring osd.7 (monmap changed)... 2024-06-21T01:08:22.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:22 smithi027 ceph-mon[90235]: Reconfiguring daemon osd.7 on smithi120 2024-06-21T01:08:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: Reconfiguring rgw.foo.smithi120.bfbgme (monmap changed)... 2024-06-21T01:08:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:23.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[84934]: Reconfiguring daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: Reconfiguring rgw.foo.smithi120.bfbgme (monmap changed)... 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:23.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:23 smithi027 ceph-mon[90235]: Reconfiguring daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:08:24.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:24.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:24.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:24.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: Reconfiguring rgw.foo.smithi120.bfbgme (monmap changed)... 2024-06-21T01:08:24.029 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:08:24.030 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:24.030 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:23 smithi120 ceph-mon[67716]: Reconfiguring daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:08:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:08:25.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:08:25.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:08:25.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:08:25.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:08:25.489 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:08:25.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:08:25.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:25.494 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:25.495 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mds 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:08:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nfs 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all grafana 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all loki 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all promtail 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:08:26.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:26 smithi120 ceph-mon[67716]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mds 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:08:26.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nfs 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all grafana 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all loki 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all promtail 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Finalizing container_image settings 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: Upgrade: Complete! 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[90235]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:26.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mds 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nfs 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all node-exporter 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all prometheus 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all alertmanager 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all grafana 2024-06-21T01:08:26.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all loki 2024-06-21T01:08:26.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all promtail 2024-06-21T01:08:26.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Finalizing container_image settings 2024-06-21T01:08:26.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: Upgrade: Complete! 2024-06-21T01:08:26.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:26 smithi027 ceph-mon[84934]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:27.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:27 smithi120 ceph-mon[67716]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:27.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:27 smithi027 ceph-mon[84934]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:27.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:27 smithi027 ceph-mon[90235]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:28.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:28] "GET /metrics HTTP/1.1" 200 37938 "" "Prometheus/2.43.0" 2024-06-21T01:08:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:29 smithi120 ceph-mon[67716]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:29 smithi120 ceph-mon[67716]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:29.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[84934]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:29.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:29.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[84934]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:29.716 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[90235]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:29.717 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:29.717 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:29 smithi027 ceph-mon[90235]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:29.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:08:31.473 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:31 smithi027 ceph-mon[84934]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:31.473 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:31 smithi027 ceph-mon[90235]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:31 smithi120 ceph-mon[67716]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:31.942 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:08:31.942 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 33s ago 25m 19.5M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:08:31.942 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (2m) 53s ago 25m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:08:31.942 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (4m) 33s ago 24m 54.0M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:08:31.942 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (4m) 53s ago 29m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (14m) 33s ago 31m 489M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (64s) 33s ago 31m 42.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (115s) 53s ago 30m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (43s) 33s ago 30m 19.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 33s ago 26m 8544k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (3m) 53s ago 26m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (29m) 33s ago 29m 55.1M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (29m) 33s ago 29m 58.8M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (28m) 33s ago 28m 49.5M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (28m) 33s ago 28m 54.4M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (28m) 53s ago 28m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (27m) 53s ago 27m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (27m) 53s ago 27m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:08:31.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (26m) 53s ago 26m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:08:31.944 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (3m) 53s ago 26m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:08:31.944 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (24m) 33s ago 24m 80.6M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:08:31.944 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (24m) 53s ago 24m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:08:32.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:32 smithi027 ceph-mon[84934]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:32.605 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:32 smithi027 ceph-mon[90235]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:32.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:32 smithi120 ceph-mon[67716]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:32.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-06-21T01:08:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:33 smithi120 ceph-mon[67716]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:33.712 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:33 smithi027 ceph-mon[84934]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:33.712 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:33 smithi027 ceph-mon[90235]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:34.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:34.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:34.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:34.895 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:08:35.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:35 smithi027 ceph-mon[84934]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:35.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:35 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/723679129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:35.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:35 smithi027 ceph-mon[90235]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:35.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:35 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/723679129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:35.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-06-21T01:08:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:35 smithi120 ceph-mon[67716]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:35 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/723679129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:37.545 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)" 2024-06-21T01:08:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:37 smithi027 ceph-mon[84934]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:37.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:37 smithi027 ceph-mon[90235]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:37.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:37 smithi120 ceph-mon[67716]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:38.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-06-21T01:08:38.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:38 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2669231389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:38.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:38 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2669231389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:38.738 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:38] "GET /metrics HTTP/1.1" 200 37934 "" "Prometheus/2.43.0" 2024-06-21T01:08:38.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:38 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2669231389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:39.676 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:39 smithi027 ceph-mon[84934]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:39.676 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:39.676 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:39 smithi027 ceph-mon[90235]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:39.676 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:39.875 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:39 smithi120 ceph-mon[67716]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:39.876 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:41.553 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:41 smithi027 ceph-mon[84934]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:41.553 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:41 smithi027 ceph-mon[84934]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:41.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:41 smithi027 ceph-mon[90235]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:41.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:41 smithi027 ceph-mon[90235]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:41.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:41 smithi120 ceph-mon[67716]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:41.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:41 smithi120 ceph-mon[67716]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:42.511 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:08:43.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:08:43.705 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:43 smithi027 ceph-mon[84934]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:43.705 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:43 smithi027 ceph-mon[90235]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:43.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:43 smithi120 ceph-mon[67716]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:45.420 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:08:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:08:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:08:45.421 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:08:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:45 smithi027 ceph-mon[84934]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:45 smithi027 ceph-mon[90235]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:45 smithi120 ceph-mon[67716]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:46.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:08:46.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:46 smithi027 ceph-mon[84934]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:46.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:46 smithi027 ceph-mon[90235]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:46.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:46 smithi120 ceph-mon[67716]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:47.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:47 smithi027 ceph-mon[84934]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:47.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:47 smithi027 ceph-mon[90235]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:47.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:47 smithi120 ceph-mon[67716]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:08:48.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:48 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3966441814' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:08:48.640 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:48 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3966441814' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:08:48.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:48 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3966441814' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:08:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:48] "GET /metrics HTTP/1.1" 200 37934 "" "Prometheus/2.43.0" 2024-06-21T01:08:49.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-06-21T01:08:49.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[84934]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:49.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:49.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:49.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[90235]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:49.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:49.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:49 smithi120 ceph-mon[67716]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:08:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:08:51.697 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:51 smithi027 ceph-mon[84934]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:51.697 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:51 smithi027 ceph-mon[84934]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:51.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:51 smithi027 ceph-mon[90235]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:51.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:51 smithi027 ceph-mon[90235]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:51 smithi120 ceph-mon[67716]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:51 smithi120 ceph-mon[67716]: from='client.64144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:52.899 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:53.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:08:54.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:54.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:54.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:08:54.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:55.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:54 smithi027 ceph-mon[84934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:55.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:54 smithi027 ceph-mon[90235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:55.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:54 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:08:55.932 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:08:56.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:55 smithi027 ceph-mon[84934]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:56.203 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:55 smithi027 ceph-mon[90235]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:56.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:55 smithi120 ceph-mon[67716]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:08:56.700 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:08:56.700 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 58s ago 25m 19.5M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:08:56.700 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (3m) 77s ago 25m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:08:56.700 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (4m) 58s ago 24m 54.0M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:08:56.700 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (4m) 77s ago 30m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (14m) 58s ago 31m 489M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (89s) 58s ago 32m 42.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (2m) 77s ago 30m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (68s) 58s ago 30m 19.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 58s ago 26m 8544k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (4m) 77s ago 26m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (30m) 58s ago 30m 55.1M 2992M 17.2.0 e1d6a67b021e 1ec969c2c27d 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (29m) 58s ago 29m 58.8M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (29m) 58s ago 29m 49.5M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (28m) 58s ago 28m 54.4M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (28m) 77s ago 28m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (28m) 77s ago 28m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (27m) 77s ago 27m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:08:56.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (27m) 77s ago 27m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:08:56.702 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (3m) 77s ago 26m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:08:56.702 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (25m) 58s ago 25m 80.6M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:08:56.702 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (24m) 77s ago 24m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:08:57.132 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:08:57.132 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:08:57.132 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-21T01:08:57.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 5 2024-06-21T01:08:57.134 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:08:57.134 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:08:57.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:56 smithi027 ceph-mon[84934]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:57.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:56 smithi027 ceph-mon[90235]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:57.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:56 smithi120 ceph-mon[67716]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:57.501 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:08:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:08:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:08:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-06-21T01:08:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:08:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/8 daemons upgraded", 2024-06-21T01:08:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:08:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:08:57.502 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:08:58.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:58.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:58 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2254923906' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:58.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2254923906' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='client.64156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:08:58.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='client.54228 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:58.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:58.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2254923906' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:08:58.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:08:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:08:58] "GET /metrics HTTP/1.1" 200 37935 "" "Prometheus/2.43.0" 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: Upgrade: osd.0 is safe to restart 2024-06-21T01:08:59.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:59.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:59.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:59.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:08:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:59.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mgr 2024-06-21T01:08:59.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:59.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:59.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: Upgrade: osd.0 is safe to restart 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mgr 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: Upgrade: osd.0 is safe to restart 2024-06-21T01:08:59.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:08:59.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:08:59.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-21T01:08:59.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:08:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:00.359 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[84934]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:00.359 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[84934]: Upgrade: Updating osd.0 2024-06-21T01:09:00.359 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[84934]: Deploying daemon osd.0 on smithi027 2024-06-21T01:09:00.360 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:00.360 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[90235]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:00.360 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[90235]: Upgrade: Updating osd.0 2024-06-21T01:09:00.360 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[90235]: Deploying daemon osd.0 on smithi027 2024-06-21T01:09:00.360 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:00 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:00 smithi120 ceph-mon[67716]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:00 smithi120 ceph-mon[67716]: Upgrade: Updating osd.0 2024-06-21T01:09:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:00 smithi120 ceph-mon[67716]: Deploying daemon osd.0 on smithi027 2024-06-21T01:09:00.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:00 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:01.159 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:01 smithi027 ceph-mon[84934]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:01.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:01 smithi027 ceph-mon[90235]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:01.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:01 smithi120 ceph-mon[67716]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:01.765 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:01 smithi027 systemd[1]: Stopping Ceph osd.0 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:09:02.114 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[32677]: 2024-06-21T01:09:01.766+0000 7ff98a15a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:09:02.114 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[32677]: 2024-06-21T01:09:01.766+0000 7ff98a15a700 -1 osd.0 113 *** Got signal Terminated *** 2024-06-21T01:09:02.114 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[32677]: 2024-06-21T01:09:01.766+0000 7ff98a15a700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:09:02.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:02 smithi120 ceph-mon[67716]: osd.0 marked itself down and dead 2024-06-21T01:09:02.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:02 smithi027 ceph-mon[84934]: osd.0 marked itself down and dead 2024-06-21T01:09:02.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:02 smithi027 ceph-mon[90235]: osd.0 marked itself down and dead 2024-06-21T01:09:02.814 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:02 smithi027 podman[96829]: 2024-06-21 01:09:02.515634229 +0000 UTC m=+0.851839466 container died 1ec969c2c27d4209c8a5bec14166f193304310454fc9243f8440aaa48f9c7f27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, com.redhat.component=centos-stream-container, distribution-scope=public, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-21T01:09:03.115 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:02 smithi027 podman[96829]: 2024-06-21 01:09:02.907367374 +0000 UTC m=+1.243572606 container cleanup 1ec969c2c27d4209c8a5bec14166f193304310454fc9243f8440aaa48f9c7f27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_CLEAN=True, distribution-scope=public, ceph=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, release=754) 2024-06-21T01:09:03.115 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:02 smithi027 bash[96829]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0 2024-06-21T01:09:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:03 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:03 smithi120 ceph-mon[67716]: osdmap e114: 8 total, 7 up, 8 in 2024-06-21T01:09:03.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:03 smithi120 ceph-mon[67716]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:03.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:03.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[90235]: osdmap e114: 8 total, 7 up, 8 in 2024-06-21T01:09:03.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[90235]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:03.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:03.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[84934]: osdmap e114: 8 total, 7 up, 8 in 2024-06-21T01:09:03.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:03 smithi027 ceph-mon[84934]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:03.479 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96846]: 2024-06-21 01:09:03.192512955 +0000 UTC m=+0.668360741 container remove 1ec969c2c27d4209c8a5bec14166f193304310454fc9243f8440aaa48f9c7f27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8) 2024-06-21T01:09:03.852 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96935]: 2024-06-21 01:09:03.446049264 +0000 UTC m=+0.017950326 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:03.852 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96935]: 2024-06-21 01:09:03.585218512 +0000 UTC m=+0.157119602 container create 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-06-21T01:09:04.124 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96935]: 2024-06-21 01:09:03.85507871 +0000 UTC m=+0.426979783 container init 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:09:04.124 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96935]: 2024-06-21 01:09:03.859120657 +0000 UTC m=+0.431021706 container start 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-21T01:09:04.124 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:03 smithi027 podman[96935]: 2024-06-21 01:09:03.988006837 +0000 UTC m=+0.559907893 container attach 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:09:04.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:04 smithi027 ceph-mon[84934]: osdmap e115: 8 total, 7 up, 8 in 2024-06-21T01:09:04.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:04 smithi027 ceph-mon[90235]: osdmap e115: 8 total, 7 up, 8 in 2024-06-21T01:09:04.478 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:04 smithi027 podman[96935]: 2024-06-21 01:09:04.127052304 +0000 UTC m=+0.698953372 container died 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-06-21T01:09:04.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:04 smithi120 ceph-mon[67716]: osdmap e115: 8 total, 7 up, 8 in 2024-06-21T01:09:05.223 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:04 smithi027 podman[96958]: 2024-06-21 01:09:04.775683577 +0000 UTC m=+0.641930937 container remove 23bd40fb5951bb9df72e54eb5de0e64615caa0501109f7127d7284bf0848ca39 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:09:05.223 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:04 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.0.service: Deactivated successfully. 2024-06-21T01:09:05.223 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:04 smithi027 systemd[1]: Stopped Ceph osd.0 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:09:05.223 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:04 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.0.service: Consumed 15.073s CPU time. 2024-06-21T01:09:05.223 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 systemd[1]: Starting Ceph osd.0 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:09:05.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[84934]: pgmap v50: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:05.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:05.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:05.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[90235]: pgmap v50: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:05.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:05.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:05.479 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.127027813 +0000 UTC m=+0.018089329 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:05.479 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.247250439 +0000 UTC m=+0.138311981 container create fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS) 2024-06-21T01:09:05.479 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.455061525 +0000 UTC m=+0.346123048 container init fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:09:05.479 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.458619403 +0000 UTC m=+0.349680912 container start fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-06-21T01:09:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:05 smithi120 ceph-mon[67716]: pgmap v50: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:05.836 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.529600672 +0000 UTC m=+0.420662185 container attach fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-21T01:09:06.229 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-21T01:09:06.230 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate[97072]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-21T01:09:06.230 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 bash[97058]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-21T01:09:06.230 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:05 smithi027 podman[97058]: 2024-06-21 01:09:05.893159597 +0000 UTC m=+0.784221101 container died fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-06-21T01:09:06.632 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:06 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:06.632 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:06 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:06.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:06 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 21 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:06.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:06 smithi027 podman[97205]: 2024-06-21 01:09:06.63496518 +0000 UTC m=+0.735904988 container remove fab7c1ffd3dc3445e10447db03b5b982cb67fc2beffe1bd055c4265334ca0bf7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-21T01:09:06.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:06 smithi027 podman[97267]: 2024-06-21 01:09:06.764044109 +0000 UTC m=+0.018811539 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:06.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:06 smithi027 podman[97267]: 2024-06-21 01:09:06.868258106 +0000 UTC m=+0.123025520 container create fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux ) 2024-06-21T01:09:07.394 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 podman[97267]: 2024-06-21 01:09:07.120891926 +0000 UTC m=+0.375659360 container init fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:09:07.395 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 podman[97267]: 2024-06-21 01:09:07.124710148 +0000 UTC m=+0.379477569 container start fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81) 2024-06-21T01:09:07.395 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-osd[97286]: -- 172.21.15.27:0/1738320794 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a212bd9a40 con 0x55a212bb7c00 2024-06-21T01:09:07.395 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 bash[97267]: fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec 2024-06-21T01:09:07.395 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 systemd[1]: Started Ceph osd.0 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:09:07.673 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[84934]: pgmap v51: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[90235]: pgmap v51: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:07 smithi120 ceph-mon[67716]: pgmap v51: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:07 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:07 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:07 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:07 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:07.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:07 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:09:07.675+0000 7f68b7576640 -1 Falling back to public interface 2024-06-21T01:09:08.791 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:08] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-06-21T01:09:09.678 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:09 smithi027 ceph-mon[90235]: pgmap v52: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:09.678 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:09.678 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:09 smithi027 ceph-mon[84934]: pgmap v52: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:09.678 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:09 smithi120 ceph-mon[67716]: pgmap v52: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:10.417 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:09:10.393+0000 7f68b7576640 -1 osd.0 0 read_superblock omap replica is missing. 2024-06-21T01:09:10.728 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:10 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:09:10.419+0000 7f68b7576640 -1 osd.0 113 log_to_monitors true 2024-06-21T01:09:11.728 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:09:11.448+0000 7f68af321640 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:09:11.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[84934]: pgmap v53: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:11.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[84934]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:11.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[84934]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:11.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[90235]: pgmap v53: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:11.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[90235]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:11.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:11 smithi027 ceph-mon[90235]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:11 smithi120 ceph-mon[67716]: pgmap v53: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:11 smithi120 ceph-mon[67716]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:11 smithi120 ceph-mon[67716]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: osdmap e116: 8 total, 7 up, 8 in 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: osdmap e116: 8 total, 7 up, 8 in 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: osdmap e116: 8 total, 7 up, 8 in 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763] boot 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: osdmap e117: 8 total, 8 up, 8 in 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[84934]: osdmap e118: 8 total, 8 up, 8 in 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:13.652 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:13.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763] boot 2024-06-21T01:09:13.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: osdmap e117: 8 total, 8 up, 8 in 2024-06-21T01:09:13.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:09:13.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:13 smithi027 ceph-mon[90235]: osdmap e118: 8 total, 8 up, 8 in 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: pgmap v55: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: osd.0 [v2:172.21.15.27:6802/201648763,v1:172.21.15.27:6803/201648763] boot 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: osdmap e117: 8 total, 8 up, 8 in 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-21T01:09:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:13 smithi120 ceph-mon[67716]: osdmap e118: 8 total, 8 up, 8 in 2024-06-21T01:09:15.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:15 smithi027 ceph-mon[84934]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-06-21T01:09:15.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:15 smithi027 ceph-mon[90235]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-06-21T01:09:15.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:15 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:09:15.637+0000 7f3f47d87640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:09:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:15 smithi120 ceph-mon[67716]: pgmap v58: 161 pgs: 37 active+undersized, 21 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:16 smithi120 ceph-mon[67716]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[84934]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:16 smithi027 ceph-mon[90235]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:09:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:17 smithi120 ceph-mon[67716]: pgmap v59: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-06-21T01:09:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:17 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:17.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:17 smithi027 ceph-mon[84934]: pgmap v59: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-06-21T01:09:17.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:17 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:17.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:17 smithi027 ceph-mon[90235]: pgmap v59: 161 pgs: 13 active+undersized, 8 active+undersized+degraded, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/723 objects degraded (4.703%) 2024-06-21T01:09:17.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:17 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:18.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:18 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded) 2024-06-21T01:09:18.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:18 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:09:18.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:18 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded) 2024-06-21T01:09:18.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:18 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:09:18.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:18] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-06-21T01:09:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:18 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/723 objects degraded (4.703%), 8 pgs degraded) 2024-06-21T01:09:18.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:18 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:09:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:19 smithi120 ceph-mon[67716]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-06-21T01:09:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:19.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[90235]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-06-21T01:09:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:19.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[84934]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-06-21T01:09:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:19.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:21 smithi120 ceph-mon[67716]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:21.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:21 smithi027 ceph-mon[90235]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:21.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:21 smithi027 ceph-mon[84934]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:23.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:23 smithi120 ceph-mon[67716]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:09:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:23 smithi027 ceph-mon[84934]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:09:23.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:23 smithi027 ceph-mon[90235]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:09:25.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:25 smithi120 ceph-mon[67716]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-06-21T01:09:25.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:25.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:25 smithi027 ceph-mon[84934]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-06-21T01:09:25.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:25.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:25 smithi027 ceph-mon[90235]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-06-21T01:09:25.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:27.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:27 smithi120 ceph-mon[67716]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:27.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:27 smithi027 ceph-mon[84934]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:27.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:27 smithi027 ceph-mon[90235]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:27.888 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:09:28.573 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:09:28.573 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 16s ago 26m 19.7M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (3m) 109s ago 26m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (5m) 16s ago 25m 55.9M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (5m) 109s ago 30m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (15m) 16s ago 32m 498M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 16s ago 32m 44.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:09:28.574 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (2m) 109s ago 31m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (100s) 16s ago 31m 32.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 16s ago 27m 8376k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (4m) 109s ago 26m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (21s) 16s ago 30m 14.3M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (30m) 16s ago 30m 59.5M 2992M 17.2.0 e1d6a67b021e bd76eaf39dc1 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (29m) 16s ago 29m 50.0M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (29m) 16s ago 29m 55.0M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:09:28.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (28m) 109s ago 28m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (28m) 109s ago 28m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (28m) 109s ago 28m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (27m) 109s ago 27m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (4m) 109s ago 27m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (25m) 16s ago 25m 80.9M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:09:28.579 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (25m) 109s ago 25m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:09:28.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:28 smithi120 ceph-mon[67716]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:28.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:28 smithi027 ceph-mon[84934]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:28.640 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:28 smithi027 ceph-mon[90235]: from='client.54252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:28] "GET /metrics HTTP/1.1" 200 38005 "" "Prometheus/2.43.0" 2024-06-21T01:09:28.991 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:09:28.991 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:09:28.992 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 1 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:09:28.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:09:28.994 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:09:28.994 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-06-21T01:09:28.994 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 6 2024-06-21T01:09:28.994 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:09:28.994 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "1/8 daemons upgraded", 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:09:29.336 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:09:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:29 smithi120 ceph-mon[67716]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:29 smithi120 ceph-mon[67716]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:29 smithi120 ceph-mon[67716]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:29 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2178592121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[84934]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[84934]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[84934]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2178592121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:29.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[90235]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[90235]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[90235]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:29.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:29 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2178592121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:30.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:30 smithi120 ceph-mon[67716]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:30.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:30 smithi027 ceph-mon[84934]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:30.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:30 smithi027 ceph-mon[90235]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:09:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:31 smithi120 ceph-mon[67716]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:31 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:31 smithi120 ceph-mon[67716]: Upgrade: osd.1 is safe to restart 2024-06-21T01:09:31.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[84934]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:31.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[84934]: Upgrade: osd.1 is safe to restart 2024-06-21T01:09:31.675 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[90235]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:09:31.675 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.675 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-21T01:09:31.675 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:31 smithi027 ceph-mon[90235]: Upgrade: osd.1 is safe to restart 2024-06-21T01:09:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[84934]: Upgrade: Updating osd.1 2024-06-21T01:09:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:09:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[90235]: Upgrade: Updating osd.1 2024-06-21T01:09:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:09:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:33 smithi120 ceph-mon[67716]: Upgrade: Updating osd.1 2024-06-21T01:09:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-21T01:09:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:34.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:34 smithi027 ceph-mon[90235]: Deploying daemon osd.1 on smithi027 2024-06-21T01:09:34.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:34 smithi027 ceph-mon[90235]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:34.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:34 smithi027 ceph-mon[84934]: Deploying daemon osd.1 on smithi027 2024-06-21T01:09:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:34 smithi027 ceph-mon[84934]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:34.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:34 smithi120 ceph-mon[67716]: Deploying daemon osd.1 on smithi027 2024-06-21T01:09:34.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:34 smithi120 ceph-mon[67716]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:35.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-mon[84934]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:35.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:35.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-mon[90235]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:35.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:35.388 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:35 smithi027 systemd[1]: Stopping Ceph osd.1 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:09:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:35 smithi120 ceph-mon[67716]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:09:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:35.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[35622]: 2024-06-21T01:09:35.389+0000 7f3e75cf6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:09:35.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[35622]: 2024-06-21T01:09:35.389+0000 7f3e75cf6700 -1 osd.1 118 *** Got signal Terminated *** 2024-06-21T01:09:35.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[35622]: 2024-06-21T01:09:35.389+0000 7f3e75cf6700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:09:36.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:36 smithi027 ceph-mon[84934]: osd.1 marked itself down and dead 2024-06-21T01:09:36.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:36 smithi027 ceph-mon[90235]: osd.1 marked itself down and dead 2024-06-21T01:09:36.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:36 smithi120 ceph-mon[67716]: osd.1 marked itself down and dead 2024-06-21T01:09:36.906 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:36 smithi027 podman[101951]: 2024-06-21 01:09:36.628316994 +0000 UTC m=+1.356988195 container died bd76eaf39dc14387ac9d1b7c962590bc3a2b9807084759222282609eec92e133 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, release=754, architecture=x86_64, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux ) 2024-06-21T01:09:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[84934]: osdmap e119: 8 total, 7 up, 8 in 2024-06-21T01:09:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[84934]: pgmap v70: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[90235]: osdmap e119: 8 total, 7 up, 8 in 2024-06-21T01:09:37.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:37 smithi027 ceph-mon[90235]: pgmap v70: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:37.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:36 smithi027 podman[101951]: 2024-06-21 01:09:36.998166022 +0000 UTC m=+1.726837211 container cleanup bd76eaf39dc14387ac9d1b7c962590bc3a2b9807084759222282609eec92e133 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux ) 2024-06-21T01:09:37.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:36 smithi027 bash[101951]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1 2024-06-21T01:09:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:37 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:09:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:37 smithi120 ceph-mon[67716]: osdmap e119: 8 total, 7 up, 8 in 2024-06-21T01:09:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:37 smithi120 ceph-mon[67716]: pgmap v70: 161 pgs: 20 stale+active+clean, 141 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:37.636 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[101967]: 2024-06-21 01:09:37.283090935 +0000 UTC m=+0.648230749 container remove bd76eaf39dc14387ac9d1b7c962590bc3a2b9807084759222282609eec92e133 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, ceph=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8) 2024-06-21T01:09:37.909 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[102058]: 2024-06-21 01:09:37.541961985 +0000 UTC m=+0.017670224 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:37.909 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[102058]: 2024-06-21 01:09:37.642322483 +0000 UTC m=+0.118030755 container create 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:09:37.909 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[102058]: 2024-06-21 01:09:37.845886955 +0000 UTC m=+0.321595195 container init 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-21T01:09:37.909 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[102058]: 2024-06-21 01:09:37.849546126 +0000 UTC m=+0.325254359 container start 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-06-21T01:09:37.909 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:37 smithi027 podman[102058]: 2024-06-21 01:09:37.911661128 +0000 UTC m=+0.387369370 container attach 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-06-21T01:09:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:38 smithi027 ceph-mon[84934]: osdmap e120: 8 total, 7 up, 8 in 2024-06-21T01:09:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:38 smithi027 ceph-mon[90235]: osdmap e120: 8 total, 7 up, 8 in 2024-06-21T01:09:38.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 podman[102058]: 2024-06-21 01:09:38.116235258 +0000 UTC m=+0.591943491 container died 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-06-21T01:09:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:38 smithi120 ceph-mon[67716]: osdmap e120: 8 total, 7 up, 8 in 2024-06-21T01:09:38.687 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:38] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-06-21T01:09:38.979 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 podman[102081]: 2024-06-21 01:09:38.690350526 +0000 UTC m=+0.564481080 container remove 83e1c9f0c22f4e7bfa9a2a396c85751e589e66efa1a7a163fd1b601969caec4e (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:09:38.979 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.1.service: Deactivated successfully. 2024-06-21T01:09:38.979 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 systemd[1]: Stopped Ceph osd.1 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:09:38.979 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.1.service: Consumed 15.747s CPU time. 2024-06-21T01:09:38.979 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:38 smithi027 systemd[1]: Starting Ceph osd.1 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:09:39.399 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-mon[90235]: pgmap v72: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:39.399 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:39.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-mon[84934]: pgmap v72: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:39.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:39.400 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.046057965 +0000 UTC m=+0.018295150 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:39.401 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.166342603 +0000 UTC m=+0.138579783 container create 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS) 2024-06-21T01:09:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:39 smithi120 ceph-mon[67716]: pgmap v72: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:09:39.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:39.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.402519002 +0000 UTC m=+0.374756192 container init 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-21T01:09:39.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.406141976 +0000 UTC m=+0.378379164 container start 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:09:39.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.477531533 +0000 UTC m=+0.449768726 container attach 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, ceph=True) 2024-06-21T01:09:40.140 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-21T01:09:40.140 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-21T01:09:40.140 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-21T01:09:40.140 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate[102195]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 bash[102181]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-21T01:09:40.141 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:39 smithi027 podman[102181]: 2024-06-21 01:09:39.84079193 +0000 UTC m=+0.813029110 container died 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-06-21T01:09:40.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:40 smithi027 ceph-mon[84934]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:09:40.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:40 smithi027 ceph-mon[90235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:09:40.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:40 smithi120 ceph-mon[67716]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:09:40.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 podman[102327]: 2024-06-21 01:09:40.431441407 +0000 UTC m=+0.584487709 container remove 37a43388fe92ff653ea2067b20b75b4b570b9fe0436235a0e041a0f5882791de (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=) 2024-06-21T01:09:40.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 podman[102390]: 2024-06-21 01:09:40.549839735 +0000 UTC m=+0.018076674 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:09:40.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 podman[102390]: 2024-06-21 01:09:40.648870856 +0000 UTC m=+0.117107783 container create eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, ceph=True) 2024-06-21T01:09:41.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 podman[102390]: 2024-06-21 01:09:40.859422605 +0000 UTC m=+0.327659541 container init eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618) 2024-06-21T01:09:41.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 podman[102390]: 2024-06-21 01:09:40.862643009 +0000 UTC m=+0.330879933 container start eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-06-21T01:09:41.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 ceph-osd[102408]: -- 172.21.15.27:0/2058382342 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562d2b319a40 con 0x562d2b2f7c00 2024-06-21T01:09:41.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 bash[102390]: eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 2024-06-21T01:09:41.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:40 smithi027 systemd[1]: Started Ceph osd.1 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:09:41.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:41 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:09:41.403+0000 7fc91d0ca640 -1 Falling back to public interface 2024-06-21T01:09:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: pgmap v73: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:09:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:42.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:42 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 20 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:42.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: pgmap v73: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:09:42.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 20 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: pgmap v73: 161 pgs: 36 active+undersized, 7 peering, 20 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:42.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:42.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:42 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 93/723 objects degraded (12.863%), 20 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:43.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:43 smithi120 ceph-mon[67716]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:09:43.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:43 smithi027 ceph-mon[84934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:09:43.408 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:43 smithi027 ceph-mon[90235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:09:44.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:44 smithi120 ceph-mon[67716]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:44.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:44 smithi027 ceph-mon[84934]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:44.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:44 smithi027 ceph-mon[90235]: pgmap v74: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:44.379 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:44 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:09:44.099+0000 7fc91d0ca640 -1 osd.1 0 read_superblock omap replica is missing. 2024-06-21T01:09:44.379 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:44 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:09:44.128+0000 7fc91d0ca640 -1 osd.1 118 log_to_monitors true 2024-06-21T01:09:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:45 smithi120 ceph-mon[67716]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:45 smithi120 ceph-mon[67716]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:45.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:45 smithi027 ceph-mon[84934]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:45.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:45 smithi027 ceph-mon[84934]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:45.478 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:09:45 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:09:45.102+0000 7fc914e75640 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:09:45.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:45 smithi027 ceph-mon[90235]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:45.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:45 smithi027 ceph-mon[90235]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: osdmap e121: 8 total, 7 up, 8 in 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:46.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:46 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: osdmap e121: 8 total, 7 up, 8 in 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: pgmap v75: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: osdmap e121: 8 total, 7 up, 8 in 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: from='osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:46 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504] boot 2024-06-21T01:09:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: osdmap e122: 8 total, 8 up, 8 in 2024-06-21T01:09:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:09:47.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[84934]: osdmap e123: 8 total, 8 up, 8 in 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504] boot 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: osdmap e122: 8 total, 8 up, 8 in 2024-06-21T01:09:47.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:09:47.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:47 smithi027 ceph-mon[90235]: osdmap e123: 8 total, 8 up, 8 in 2024-06-21T01:09:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:09:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: osd.1 [v2:172.21.15.27:6810/3088596504,v1:172.21.15.27:6811/3088596504] boot 2024-06-21T01:09:47.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: osdmap e122: 8 total, 8 up, 8 in 2024-06-21T01:09:47.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-21T01:09:47.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:47.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:47 smithi120 ceph-mon[67716]: osdmap e123: 8 total, 8 up, 8 in 2024-06-21T01:09:48.221 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:48 smithi027 ceph-mon[84934]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:48.221 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:48 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 100/723 objects degraded (13.831%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:48.221 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:48 smithi027 ceph-mon[90235]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:48.221 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:48 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 100/723 objects degraded (13.831%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:48 smithi120 ceph-mon[67716]: pgmap v78: 161 pgs: 41 active+undersized, 22 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 100/723 objects degraded (13.831%) 2024-06-21T01:09:48.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:48 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 100/723 objects degraded (13.831%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:09:48.932 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:48] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-06-21T01:09:50.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: pgmap v80: 161 pgs: 6 peering, 36 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-06-21T01:09:50.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:50.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:50.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.371 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:09:50.372 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.373 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.374 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: pgmap v80: 161 pgs: 6 peering, 36 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:09:50.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: pgmap v80: 161 pgs: 6 peering, 36 active+undersized, 21 active+undersized+degraded, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 94/723 objects degraded (13.001%) 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:09:50.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:09:50.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:09:50.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:09:50.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:51 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:09:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:51 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:09:51.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:51 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:09:51.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:09:51.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[84934]: Upgrade: Finalizing container_image settings 2024-06-21T01:09:51.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[84934]: Upgrade: Complete! 2024-06-21T01:09:51.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:09:51.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[90235]: Upgrade: Finalizing container_image settings 2024-06-21T01:09:51.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:51 smithi027 ceph-mon[90235]: Upgrade: Complete! 2024-06-21T01:09:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:52 smithi120 ceph-mon[67716]: pgmap v81: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:09:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:52 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/723 objects degraded (13.001%), 21 pgs degraded) 2024-06-21T01:09:52.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:52 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:09:52.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[84934]: pgmap v81: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:09:52.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/723 objects degraded (13.001%), 21 pgs degraded) 2024-06-21T01:09:52.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:09:52.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[90235]: pgmap v81: 161 pgs: 6 peering, 155 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:09:52.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 94/723 objects degraded (13.001%), 21 pgs degraded) 2024-06-21T01:09:52.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:52 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:09:54.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:54 smithi120 ceph-mon[67716]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:09:54.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:54 smithi027 ceph-mon[84934]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:09:54.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:54 smithi027 ceph-mon[90235]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:09:55.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:55 smithi027 ceph-mon[84934]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-21T01:09:55.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:55.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:55 smithi027 ceph-mon[90235]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-21T01:09:55.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:55 smithi120 ceph-mon[67716]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 621 B/s rd, 0 op/s 2024-06-21T01:09:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:09:57.668 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:57 smithi120 ceph-mon[67716]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:57.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:57 smithi027 ceph-mon[84934]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:57.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:57 smithi027 ceph-mon[90235]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:09:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:09:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:09:58] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.43.0" 2024-06-21T01:09:59.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:59 smithi027 ceph-mon[90235]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-06-21T01:09:59.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:09:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:59.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:59 smithi027 ceph-mon[84934]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-06-21T01:09:59.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:09:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:09:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:59 smithi120 ceph-mon[67716]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 911 B/s rd, 0 op/s 2024-06-21T01:09:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:09:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:00.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[84934]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[84934]: overall HEALTH_OK 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[90235]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:00.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:00 smithi027 ceph-mon[90235]: overall HEALTH_OK 2024-06-21T01:10:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:00 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:00 smithi120 ceph-mon[67716]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:00 smithi120 ceph-mon[67716]: overall HEALTH_OK 2024-06-21T01:10:01.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:01 smithi027 ceph-mon[84934]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:01.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:01 smithi027 ceph-mon[90235]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:01 smithi120 ceph-mon[67716]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 17s ago 26m 19.7M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (4m) 2m ago 26m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (5m) 17s ago 25m 57.9M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (5m) 2m ago 31m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (15m) 17s ago 33m 500M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 17s ago 33m 49.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (3m) 2m ago 31m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (2m) 17s ago 31m 35.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:10:02.549 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (5m) 17s ago 27m 9005k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (5m) 2m ago 27m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (55s) 17s ago 31m 46.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (21s) 17s ago 30m 11.0M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (30m) 17s ago 30m 51.3M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (29m) 17s ago 29m 55.5M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (29m) 2m ago 29m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (29m) 2m ago 29m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (28m) 2m ago 28m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (28m) 2m ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (5m) 2m ago 27m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (26m) 17s ago 26m 81.0M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:10:02.550 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (26m) 2m ago 26m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:10:03.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-06-21T01:10:03.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:03 smithi027 ceph-mon[84934]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:03.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:03 smithi027 ceph-mon[84934]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:03.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:03 smithi027 ceph-mon[90235]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:03.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:03 smithi027 ceph-mon[90235]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:03 smithi120 ceph-mon[67716]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:03 smithi120 ceph-mon[67716]: from='client.54294 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:05.465 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[84934]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[90235]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:05.466 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:05 smithi120 ceph-mon[67716]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:06.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-06-21T01:10:06.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:06 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3450423680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:06.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:06 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3450423680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:06.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:06 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3450423680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:07.557 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:07 smithi027 ceph-mon[84934]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:07.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:07 smithi027 ceph-mon[90235]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:07.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:07 smithi120 ceph-mon[67716]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:08] "GET /metrics HTTP/1.1" 200 38089 "" "Prometheus/2.43.0" 2024-06-21T01:10:09.583 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[84934]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:09.583 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[84934]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:09.583 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:09.583 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[90235]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:09.583 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[90235]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:09.584 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:09 smithi120 ceph-mon[67716]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:09 smithi120 ceph-mon[67716]: from='client.54306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:10.374 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:10:11.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:10:11.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:11 smithi120 ceph-mon[67716]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:11.685 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:11 smithi027 ceph-mon[84934]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:11.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:11 smithi027 ceph-mon[90235]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:10:13.434 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:10:13.435 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:13 smithi027 ceph-mon[84934]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:13.435 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:13 smithi027 ceph-mon[90235]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:13.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:13 smithi120 ceph-mon[67716]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:14.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:10:14.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:14 smithi120 ceph-mon[67716]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:14.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:14 smithi027 ceph-mon[84934]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:14.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:14 smithi027 ceph-mon[90235]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:15.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:15 smithi027 ceph-mon[84934]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:15.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:15 smithi027 ceph-mon[90235]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:15.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:15 smithi120 ceph-mon[67716]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:16.222 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:10:16.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:16 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/842818187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:10:16.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:16 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/842818187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:10:16.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-06-21T01:10:16.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:16 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/842818187' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:10:17.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:17 smithi027 ceph-mon[84934]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:17.408 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:17 smithi027 ceph-mon[90235]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:17 smithi120 ceph-mon[67716]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:18.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:18] "GET /metrics HTTP/1.1" 200 38089 "" "Prometheus/2.43.0" 2024-06-21T01:10:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[84934]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[84934]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[90235]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[90235]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:19 smithi120 ceph-mon[67716]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:19 smithi120 ceph-mon[67716]: from='client.54324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:21.127 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[84934]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[90235]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:21.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:21.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:10:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:21 smithi120 ceph-mon[67716]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:21.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:22.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[84934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[84934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[90235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:22.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:22 smithi027 ceph-mon[90235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:22.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:22 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:22.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:22.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:22.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:22.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:22 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:10:23.730 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:10:23.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:23 smithi027 ceph-mon[84934]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:23.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:23 smithi027 ceph-mon[90235]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:23 smithi120 ceph-mon[67716]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 38s ago 27m 19.7M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (4m) 2m ago 27m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (5m) 38s ago 26m 57.9M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (6m) 2m ago 31m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:10:24.408 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (16m) 38s ago 33m 500M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 38s ago 33m 49.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (3m) 2m ago 32m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (2m) 38s ago 32m 35.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (5m) 38s ago 27m 9005k - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (5m) 2m ago 27m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (77s) 38s ago 31m 46.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (43s) 38s ago 31m 11.0M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (30m) 38s ago 30m 51.3M 2992M 17.2.0 e1d6a67b021e 37f02e17820b 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (30m) 38s ago 30m 55.5M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (29m) 2m ago 29m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (29m) 2m ago 29m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:10:24.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (29m) 2m ago 29m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:10:24.410 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (28m) 2m ago 28m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:10:24.410 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (5m) 2m ago 28m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:10:24.410 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (26m) 38s ago 26m 81.0M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:10:24.410 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (26m) 2m ago 26m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:10:24.832 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:10:24.832 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:10:24.833 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:10:24.834 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-06-21T01:10:24.834 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 7 2024-06-21T01:10:24.834 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:10:24.834 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:10:24.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:24 smithi027 ceph-mon[90235]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:24.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:24 smithi027 ceph-mon[90235]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:24 smithi027 ceph-mon[84934]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:24 smithi027 ceph-mon[84934]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:24 smithi120 ceph-mon[67716]: from='client.64264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:24 smithi120 ceph-mon[67716]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:25.189 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:10:25.189 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", 2024-06-21T01:10:25.189 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:10:25.189 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-06-21T01:10:25.190 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:10:25.190 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "", 2024-06-21T01:10:25.190 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd image", 2024-06-21T01:10:25.190 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:10:25.190 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/216375789' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:25.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/216375789' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:25.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:25 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='client.54342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/216375789' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:26.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:25 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:26.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:26.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mgr 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[90235]: Upgrade: osd.2 is safe to restart 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:10:26.958 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mgr 2024-06-21T01:10:26.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:10:26.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:10:26.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:26.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:26 smithi027 ceph-mon[84934]: Upgrade: osd.2 is safe to restart 2024-06-21T01:10:27.017 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-21T01:10:27.018 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:26 smithi120 ceph-mon[67716]: Upgrade: osd.2 is safe to restart 2024-06-21T01:10:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: Upgrade: Updating osd.2 2024-06-21T01:10:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[84934]: Deploying daemon osd.2 on smithi027 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: Upgrade: Updating osd.2 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:27 smithi027 ceph-mon[90235]: Deploying daemon osd.2 on smithi027 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: Upgrade: Updating osd.2 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:27 smithi120 ceph-mon[67716]: Deploying daemon osd.2 on smithi027 2024-06-21T01:10:28.885 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:28] "GET /metrics HTTP/1.1" 200 38092 "" "Prometheus/2.43.0" 2024-06-21T01:10:30.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:29 smithi027 ceph-mon[84934]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:30.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:30.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:29 smithi027 ceph-mon[90235]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:30.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:29 smithi120 ceph-mon[67716]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:30.420 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:30 smithi027 systemd[1]: Stopping Ceph osd.2 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:10:30.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[38526]: 2024-06-21T01:10:30.421+0000 7fc13b97b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:10:30.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[38526]: 2024-06-21T01:10:30.421+0000 7fc13b97b700 -1 osd.2 123 *** Got signal Terminated *** 2024-06-21T01:10:30.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[38526]: 2024-06-21T01:10:30.421+0000 7fc13b97b700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:10:31.344 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:30 smithi027 ceph-mon[84934]: osd.2 marked itself down and dead 2024-06-21T01:10:31.345 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:31 smithi027 ceph-mon[90235]: osd.2 marked itself down and dead 2024-06-21T01:10:31.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:30 smithi120 ceph-mon[67716]: osd.2 marked itself down and dead 2024-06-21T01:10:31.620 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:31 smithi027 podman[108252]: 2024-06-21 01:10:31.347057712 +0000 UTC m=+1.048724198 container died 37f02e17820b80023386fb77a232ccf461d80eaa64467d4c8d20b3ca3ffeff25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-21T01:10:31.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:31 smithi027 podman[108252]: 2024-06-21 01:10:31.688346044 +0000 UTC m=+1.390012530 container cleanup 37f02e17820b80023386fb77a232ccf461d80eaa64467d4c8d20b3ca3ffeff25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, ceph=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, distribution-scope=public, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-21T01:10:31.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:31 smithi027 bash[108252]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2 2024-06-21T01:10:31.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:31 smithi027 podman[108269]: 2024-06-21 01:10:31.864694807 +0000 UTC m=+0.507702196 container remove 37f02e17820b80023386fb77a232ccf461d80eaa64467d4c8d20b3ca3ffeff25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-06-21T01:10:32.337 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[84934]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:32.337 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:10:32.338 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[84934]: osdmap e124: 8 total, 7 up, 8 in 2024-06-21T01:10:32.338 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[90235]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:32.338 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:10:32.338 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:32 smithi027 ceph-mon[90235]: osdmap e124: 8 total, 7 up, 8 in 2024-06-21T01:10:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:32 smithi120 ceph-mon[67716]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:32 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:10:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:32 smithi120 ceph-mon[67716]: osdmap e124: 8 total, 7 up, 8 in 2024-06-21T01:10:32.615 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.243745867 +0000 UTC m=+0.018217755 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:10:32.615 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.365614568 +0000 UTC m=+0.140086446 container create 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:10:32.615 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.618045192 +0000 UTC m=+0.392517079 container init 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:10:32.894 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.622140481 +0000 UTC m=+0.396612370 container start 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-06-21T01:10:32.894 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.726440821 +0000 UTC m=+0.500912705 container attach 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux ) 2024-06-21T01:10:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:33 smithi027 ceph-mon[84934]: osdmap e125: 8 total, 7 up, 8 in 2024-06-21T01:10:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:33 smithi027 ceph-mon[90235]: osdmap e125: 8 total, 7 up, 8 in 2024-06-21T01:10:33.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:32 smithi027 podman[108358]: 2024-06-21 01:10:32.897277287 +0000 UTC m=+0.671749169 container died 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:10:33.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:33 smithi120 ceph-mon[67716]: osdmap e125: 8 total, 7 up, 8 in 2024-06-21T01:10:33.975 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 podman[108380]: 2024-06-21 01:10:33.530647731 +0000 UTC m=+0.623348131 container remove 3d11f7838d2f6a143b951802dd823884c9940f6ad411bbfd8c999f32288cbabe (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0) 2024-06-21T01:10:33.975 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.2.service: Deactivated successfully. 2024-06-21T01:10:33.975 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 systemd[1]: Stopped Ceph osd.2 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:10:33.975 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.2.service: Consumed 13.412s CPU time. 2024-06-21T01:10:33.976 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 systemd[1]: Starting Ceph osd.2 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:10:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-mon[84934]: pgmap v104: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-mon[84934]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:10:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-mon[90235]: pgmap v104: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-mon[90235]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:10:34.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 podman[108478]: 2024-06-21 01:10:33.880797819 +0000 UTC m=+0.018066717 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:10:34.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:33 smithi027 podman[108478]: 2024-06-21 01:10:33.981479992 +0000 UTC m=+0.118748879 container create 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:10:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:34 smithi120 ceph-mon[67716]: pgmap v104: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:34.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:34 smithi120 ceph-mon[67716]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:10:34.659 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 podman[108478]: 2024-06-21 01:10:34.275235331 +0000 UTC m=+0.412504220 container init 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux ) 2024-06-21T01:10:34.659 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 podman[108478]: 2024-06-21 01:10:34.279136976 +0000 UTC m=+0.416405865 container start 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-06-21T01:10:34.659 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 podman[108478]: 2024-06-21 01:10:34.376155828 +0000 UTC m=+0.513424733 container attach 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81) 2024-06-21T01:10:34.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-21T01:10:34.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-21T01:10:34.978 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate[108493]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 bash[108478]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-21T01:10:34.979 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:34 smithi027 podman[108478]: 2024-06-21 01:10:34.715772728 +0000 UTC m=+0.853041619 container died 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-21T01:10:35.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[84934]: pgmap v105: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:35.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:35.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:35.595 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[90235]: pgmap v105: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:35.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:35.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:35.596 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 podman[108609]: 2024-06-21 01:10:35.381020874 +0000 UTC m=+0.659188318 container remove 6fab0a06213e66cfc6e6f804fee750db577d8618cb45f3ca686b682d8212f6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, ceph=True) 2024-06-21T01:10:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:35 smithi120 ceph-mon[67716]: pgmap v105: 161 pgs: 7 peering, 10 stale+active+clean, 144 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:35.848 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 podman[108666]: 2024-06-21 01:10:35.500774985 +0000 UTC m=+0.018628264 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:10:35.848 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 podman[108666]: 2024-06-21 01:10:35.618570049 +0000 UTC m=+0.136423356 container create c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True) 2024-06-21T01:10:35.848 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 podman[108666]: 2024-06-21 01:10:35.826442167 +0000 UTC m=+0.344295459 container init c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-06-21T01:10:35.849 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 podman[108666]: 2024-06-21 01:10:35.829871196 +0000 UTC m=+0.347724468 container start c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-06-21T01:10:36.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 ceph-osd[108686]: -- 172.21.15.27:0/1694490549 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aa9a56da40 con 0x55aa9b32c000 2024-06-21T01:10:36.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 bash[108666]: c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 2024-06-21T01:10:36.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:35 smithi027 systemd[1]: Started Ceph osd.2 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:10:36.697 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:10:36.379+0000 7fc842f8b640 -1 Falling back to public interface 2024-06-21T01:10:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:36 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:36 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:36 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:36 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:36 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:38.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:37 smithi120 ceph-mon[67716]: pgmap v106: 161 pgs: 26 active+undersized, 7 peering, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:38.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:37 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:38.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:37 smithi027 ceph-mon[84934]: pgmap v106: 161 pgs: 26 active+undersized, 7 peering, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:38.386 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:37 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:38.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:37 smithi027 ceph-mon[90235]: pgmap v106: 161 pgs: 26 active+undersized, 7 peering, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:38.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:37 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:38.945 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:38] "GET /metrics HTTP/1.1" 200 38157 "" "Prometheus/2.43.0" 2024-06-21T01:10:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:39 smithi027 ceph-mon[84934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-21T01:10:39.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:39 smithi027 ceph-mon[90235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-21T01:10:39.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:10:39.083+0000 7fc842f8b640 -1 osd.2 0 read_superblock omap replica is missing. 2024-06-21T01:10:39.229 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:39 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:10:39.105+0000 7fc842f8b640 -1 osd.2 123 log_to_monitors true 2024-06-21T01:10:39.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:39 smithi120 ceph-mon[67716]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-06-21T01:10:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:40 smithi120 ceph-mon[67716]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:10:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:40 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:40.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:40 smithi120 ceph-mon[67716]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[84934]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[84934]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[90235]: pgmap v107: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:40.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-mon[90235]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-21T01:10:40.479 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:10:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:10:40.074+0000 7fc83ad36640 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: osdmap e126: 8 total, 7 up, 8 in 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T01:10:41.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: osdmap e126: 8 total, 7 up, 8 in 2024-06-21T01:10:41.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: osdmap e126: 8 total, 7 up, 8 in 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:41.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: pgmap v109: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355] boot 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: osdmap e127: 8 total, 8 up, 8 in 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:10:42.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:42 smithi120 ceph-mon[67716]: osdmap e128: 8 total, 8 up, 8 in 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: pgmap v109: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355] boot 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: osdmap e127: 8 total, 8 up, 8 in 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[84934]: osdmap e128: 8 total, 8 up, 8 in 2024-06-21T01:10:42.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: pgmap v109: 161 pgs: 32 active+undersized, 15 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 616 B/s rd, 0 op/s; 67/723 objects degraded (9.267%) 2024-06-21T01:10:42.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:10:42.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: osd.2 [v2:172.21.15.27:6818/1406836355,v1:172.21.15.27:6819/1406836355] boot 2024-06-21T01:10:42.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: osdmap e127: 8 total, 8 up, 8 in 2024-06-21T01:10:42.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-21T01:10:42.452 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:42 smithi027 ceph-mon[90235]: osdmap e128: 8 total, 8 up, 8 in 2024-06-21T01:10:43.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:43 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:43.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:43 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:43.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:43 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 67/723 objects degraded (9.267%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:10:44.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:44 smithi120 ceph-mon[67716]: pgmap v112: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:44.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:44.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:44 smithi027 ceph-mon[90235]: pgmap v112: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:44.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:44 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:44.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:44 smithi027 ceph-mon[84934]: pgmap v112: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:44.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:10:45.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:10:45.369 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.370 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:10:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:10:45.483 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:10:45.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:10:45.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:10:45.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:10:45.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:45.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:46 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:10:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:46 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:10:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:46 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:10:46.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:46 smithi120 ceph-mon[67716]: pgmap v113: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:10:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[84934]: Upgrade: Finalizing container_image settings 2024-06-21T01:10:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[84934]: Upgrade: Complete! 2024-06-21T01:10:46.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[84934]: pgmap v113: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:46.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:10:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[90235]: Upgrade: Finalizing container_image settings 2024-06-21T01:10:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[90235]: Upgrade: Complete! 2024-06-21T01:10:46.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:46 smithi027 ceph-mon[90235]: pgmap v113: 161 pgs: 5 peering, 28 active+undersized, 14 active+undersized+degraded, 114 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 61/723 objects degraded (8.437%) 2024-06-21T01:10:47.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[84934]: pgmap v114: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-21T01:10:47.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded) 2024-06-21T01:10:47.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:10:47.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[90235]: pgmap v114: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-21T01:10:47.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded) 2024-06-21T01:10:47.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:47 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:10:47.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:47 smithi120 ceph-mon[67716]: pgmap v114: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-06-21T01:10:47.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:47 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 61/723 objects degraded (8.437%), 14 pgs degraded) 2024-06-21T01:10:47.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:47 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:10:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:48] "GET /metrics HTTP/1.1" 200 38157 "" "Prometheus/2.43.0" 2024-06-21T01:10:49.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[84934]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:49.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:49.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:49.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:49.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[90235]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:49.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:49.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:49.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:49 smithi120 ceph-mon[67716]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:10:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:10:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:10:51.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:51 smithi027 ceph-mon[84934]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:10:51.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:51 smithi027 ceph-mon[90235]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:10:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:51 smithi120 ceph-mon[67716]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:10:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:53 smithi027 ceph-mon[84934]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-06-21T01:10:53.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:53 smithi027 ceph-mon[90235]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-06-21T01:10:53.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:53 smithi120 ceph-mon[67716]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-06-21T01:10:55.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:55 smithi027 ceph-mon[84934]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:55.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:55 smithi027 ceph-mon[90235]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:55 smithi120 ceph-mon[67716]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:56.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:10:56.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:56 smithi027 ceph-mon[84934]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:56.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:56 smithi027 ceph-mon[90235]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:56.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:56 smithi120 ceph-mon[67716]: from='client.64300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:57.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:57 smithi027 ceph-mon[84934]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:57.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:57 smithi027 ceph-mon[90235]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:57.767 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:57 smithi120 ceph-mon[67716]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:10:58.249 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:10:58.249 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 18s ago 27m 28.1M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:10:58.249 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (5m) 3m ago 27m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:10:58.249 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (6m) 18s ago 26m 58.1M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (6m) 3m ago 32m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (16m) 18s ago 34m 502M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (3m) 18s ago 34m 51.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (4m) 3m ago 32m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (3m) 18s ago 32m 37.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 18s ago 28m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (6m) 3m ago 28m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (111s) 18s ago 32m 46.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (77s) 18s ago 31m 43.7M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (22s) 18s ago 31m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (30m) 18s ago 30m 57.0M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (30m) 3m ago 30m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:10:58.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (30m) 3m ago 30m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:10:58.251 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (29m) 3m ago 29m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:10:58.251 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (29m) 3m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:10:58.251 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (5m) 3m ago 28m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:10:58.251 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (27m) 18s ago 27m 81.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:10:58.251 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (26m) 3m ago 26m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:10:58.952 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:10:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:10:58] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-06-21T01:10:58.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[84934]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[84934]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[90235]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[90235]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:59.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:10:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:10:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:59 smithi120 ceph-mon[67716]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:10:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:59 smithi120 ceph-mon[67716]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:10:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:10:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:01.094 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:11:01.666 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:01 smithi027 ceph-mon[90235]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:01.667 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:01 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2634680989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:01.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:01 smithi027 ceph-mon[84934]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:01.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:01 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2634680989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:01.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-06-21T01:11:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:01 smithi120 ceph-mon[67716]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:01 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2634680989' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:03.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:03 smithi027 ceph-mon[84934]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:03.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:03 smithi027 ceph-mon[90235]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:03 smithi120 ceph-mon[67716]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:04.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:04 smithi027 ceph-mon[84934]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:04.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:04.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:04 smithi027 ceph-mon[90235]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:04.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:04.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:04 smithi120 ceph-mon[67716]: from='client.64318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:04.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:05.675 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:05 smithi027 ceph-mon[84934]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:05.676 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:05 smithi027 ceph-mon[90235]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:05.744 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:11:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:05 smithi120 ceph-mon[67716]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:06.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:11:07.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:07 smithi027 ceph-mon[84934]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:07.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:07 smithi027 ceph-mon[90235]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:07 smithi120 ceph-mon[67716]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:08.604 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:11:08.605 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:11:08.931 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:08] "GET /metrics HTTP/1.1" 200 38165 "" "Prometheus/2.43.0" 2024-06-21T01:11:09.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:11:09.701 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[84934]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:09.702 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[84934]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:09.702 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:09.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[90235]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:09.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[90235]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:09.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:09 smithi120 ceph-mon[67716]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:09 smithi120 ceph-mon[67716]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:11.682 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:11:11.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:11 smithi027 ceph-mon[84934]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:11.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:11 smithi027 ceph-mon[90235]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:11 smithi120 ceph-mon[67716]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:12.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-06-21T01:11:12.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:12 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1765437269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:11:12.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:12 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1765437269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:11:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:12 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1765437269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:11:13.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:13 smithi027 ceph-mon[84934]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:13 smithi027 ceph-mon[90235]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:13 smithi120 ceph-mon[67716]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:15.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:15 smithi027 ceph-mon[84934]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:15.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:15 smithi027 ceph-mon[84934]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:15.662 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:15 smithi027 ceph-mon[90235]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:15.662 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:15 smithi027 ceph-mon[90235]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:15 smithi120 ceph-mon[67716]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:15 smithi120 ceph-mon[67716]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:16.610 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[84934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:17 smithi027 ceph-mon[90235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:17.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:17 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:11:18.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:18] "GET /metrics HTTP/1.1" 200 38165 "" "Prometheus/2.43.0" 2024-06-21T01:11:19.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[84934]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:19.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:19.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[90235]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:19 smithi120 ceph-mon[67716]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:19.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:20.148 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:11:20.835 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:11:20.835 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 40s ago 28m 28.1M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:11:20.835 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (5m) 3m ago 27m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:11:20.835 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (6m) 40s ago 27m 58.1M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (7m) 3m ago 32m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (17m) 40s ago 34m 502M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (3m) 40s ago 34m 51.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (4m) 3m ago 32m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (3m) 40s ago 33m 37.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 40s ago 28m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (6m) 3m ago 28m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (2m) 40s ago 32m 46.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (100s) 40s ago 32m 43.7M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (45s) 40s ago 31m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (31m) 40s ago 31m 57.0M 2992M 17.2.0 e1d6a67b021e 68eba76e6d70 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (30m) 3m ago 30m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (30m) 3m ago 30m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (30m) 3m ago 30m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:11:20.836 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (29m) 3m ago 29m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:11:20.837 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (6m) 3m ago 28m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:11:20.837 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (27m) 40s ago 27m 81.2M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:11:20.837 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (27m) 3m ago 27m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:11:21.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 8 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:11:21.289 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:11:21.643 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "3/8 daemons upgraded", 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:11:21.644 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mgr 2024-06-21T01:11:22.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: Upgrade: osd.3 is safe to restart 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3213727166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mgr 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:11:22.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: Upgrade: osd.3 is safe to restart 2024-06-21T01:11:22.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:21 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3213727166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='client.54408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='client.54414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:22.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-21T01:11:22.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: Upgrade: osd.3 is safe to restart 2024-06-21T01:11:22.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:21 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3213727166' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[84934]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[90235]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:23.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:23.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:11:23.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:22 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:22 smithi120 ceph-mon[67716]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-21T01:11:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:22 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:24.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[84934]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:24.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[84934]: Upgrade: Updating osd.3 2024-06-21T01:11:24.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[84934]: Deploying daemon osd.3 on smithi027 2024-06-21T01:11:24.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[90235]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:24.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[90235]: Upgrade: Updating osd.3 2024-06-21T01:11:24.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:23 smithi027 ceph-mon[90235]: Deploying daemon osd.3 on smithi027 2024-06-21T01:11:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:23 smithi120 ceph-mon[67716]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:23 smithi120 ceph-mon[67716]: Upgrade: Updating osd.3 2024-06-21T01:11:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:23 smithi120 ceph-mon[67716]: Deploying daemon osd.3 on smithi027 2024-06-21T01:11:26.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:25 smithi027 ceph-mon[84934]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:26.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:25 smithi027 ceph-mon[90235]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:26.025 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:25 smithi027 systemd[1]: Stopping Ceph osd.3 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:11:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:25 smithi120 ceph-mon[67716]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:26.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[41495]: 2024-06-21T01:11:26.026+0000 7f14094b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:11:26.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[41495]: 2024-06-21T01:11:26.026+0000 7f14094b8700 -1 osd.3 128 *** Got signal Terminated *** 2024-06-21T01:11:26.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:26 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[41495]: 2024-06-21T01:11:26.026+0000 7f14094b8700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:11:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:26 smithi027 ceph-mon[84934]: osd.3 marked itself down and dead 2024-06-21T01:11:27.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:26 smithi027 ceph-mon[90235]: osd.3 marked itself down and dead 2024-06-21T01:11:27.267 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:26 smithi120 ceph-mon[67716]: osd.3 marked itself down and dead 2024-06-21T01:11:27.487 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:27 smithi027 podman[114768]: 2024-06-21 01:11:27.440246996 +0000 UTC m=+1.539324049 container died 68eba76e6d7012e39c291961fb8c44a77d23088070eaed4b7c2593637dc88f19 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, release=754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-06-21T01:11:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[84934]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:11:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[84934]: osdmap e129: 8 total, 7 up, 8 in 2024-06-21T01:11:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[90235]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:11:28.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:27 smithi027 ceph-mon[90235]: osdmap e129: 8 total, 7 up, 8 in 2024-06-21T01:11:28.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:27 smithi027 podman[114768]: 2024-06-21 01:11:27.910112733 +0000 UTC m=+2.009189786 container cleanup 68eba76e6d7012e39c291961fb8c44a77d23088070eaed4b7c2593637dc88f19 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_CLEAN=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , release=754, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-06-21T01:11:28.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:27 smithi027 bash[114768]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3 2024-06-21T01:11:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:27 smithi120 ceph-mon[67716]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:27 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:11:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:27 smithi120 ceph-mon[67716]: osdmap e129: 8 total, 7 up, 8 in 2024-06-21T01:11:28.640 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:28 smithi027 podman[114786]: 2024-06-21 01:11:28.287194976 +0000 UTC m=+0.840100861 container remove 68eba76e6d7012e39c291961fb8c44a77d23088070eaed4b7c2593637dc88f19 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream) 2024-06-21T01:11:28.899 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:28] "GET /metrics HTTP/1.1" 200 38169 "" "Prometheus/2.43.0" 2024-06-21T01:11:28.899 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:28 smithi027 podman[114883]: 2024-06-21 01:11:28.638470557 +0000 UTC m=+0.017967779 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:11:28.899 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:28 smithi027 podman[114883]: 2024-06-21 01:11:28.763347425 +0000 UTC m=+0.142844701 container create abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618) 2024-06-21T01:11:29.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:28 smithi027 ceph-mon[84934]: osdmap e130: 8 total, 7 up, 8 in 2024-06-21T01:11:29.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:28 smithi027 ceph-mon[90235]: osdmap e130: 8 total, 7 up, 8 in 2024-06-21T01:11:29.228 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 podman[114883]: 2024-06-21 01:11:29.032701806 +0000 UTC m=+0.412199019 container init abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-06-21T01:11:29.228 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 podman[114883]: 2024-06-21 01:11:29.036397355 +0000 UTC m=+0.415894562 container start abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240618) 2024-06-21T01:11:29.228 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 podman[114883]: 2024-06-21 01:11:29.132537278 +0000 UTC m=+0.512034486 container attach abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-06-21T01:11:29.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:28 smithi120 ceph-mon[67716]: osdmap e130: 8 total, 7 up, 8 in 2024-06-21T01:11:29.591 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 podman[114883]: 2024-06-21 01:11:29.300279369 +0000 UTC m=+0.679776577 container died abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-06-21T01:11:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:29 smithi027 ceph-mon[84934]: pgmap v137: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:11:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:29 smithi027 ceph-mon[90235]: pgmap v137: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:11:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:30.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 podman[114906]: 2024-06-21 01:11:29.978220449 +0000 UTC m=+0.672124473 container remove abcd9785d3a137c2de87c81cf78a579dfc32c851da372f85313c2d5899038a29 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-06-21T01:11:30.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.3.service: Deactivated successfully. 2024-06-21T01:11:30.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 systemd[1]: Stopped Ceph osd.3 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:11:30.229 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:29 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.3.service: Consumed 17.878s CPU time. 2024-06-21T01:11:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:29 smithi120 ceph-mon[67716]: pgmap v137: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:11:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:30.566 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 systemd[1]: Starting Ceph osd.3 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:11:30.566 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 podman[115003]: 2024-06-21 01:11:30.368430003 +0000 UTC m=+0.018667041 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:11:30.566 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 podman[115003]: 2024-06-21 01:11:30.471360055 +0000 UTC m=+0.121597083 container create 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0) 2024-06-21T01:11:30.978 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 podman[115003]: 2024-06-21 01:11:30.682315836 +0000 UTC m=+0.332552877 container init 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-06-21T01:11:30.978 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 podman[115003]: 2024-06-21 01:11:30.686052851 +0000 UTC m=+0.336289879 container start 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0) 2024-06-21T01:11:30.979 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:30 smithi027 podman[115003]: 2024-06-21 01:11:30.757134479 +0000 UTC m=+0.407371521 container attach 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-06-21T01:11:31.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:31 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 67/723 objects degraded (9.267%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:31.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 67/723 objects degraded (9.267%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:31.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 67/723 objects degraded (9.267%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-21T01:11:31.416 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate[115017]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 bash[115003]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-21T01:11:31.417 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 podman[115003]: 2024-06-21 01:11:31.123263273 +0000 UTC m=+0.773500304 container died 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:11:32.157 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:32 smithi027 ceph-mon[84934]: pgmap v138: 161 pgs: 33 active+undersized, 6 stale+active+clean, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:11:32.157 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:32 smithi027 ceph-mon[90235]: pgmap v138: 161 pgs: 33 active+undersized, 6 stale+active+clean, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:11:32.157 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:31 smithi027 podman[115144]: 2024-06-21 01:11:31.811386339 +0000 UTC m=+0.682160801 container remove 7f01f8400248b266a1cfa32a38dd1b4d661e7fae6bf8746b6308a5ecee0f237f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-activate, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618) 2024-06-21T01:11:32.157 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 podman[115204]: 2024-06-21 01:11:31.935717136 +0000 UTC m=+0.018828470 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:11:32.158 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 podman[115204]: 2024-06-21 01:11:32.054293048 +0000 UTC m=+0.137404373 container create 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_CLEAN=True, ceph=True, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0) 2024-06-21T01:11:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:32 smithi120 ceph-mon[67716]: pgmap v138: 161 pgs: 33 active+undersized, 6 stale+active+clean, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 67/723 objects degraded (9.267%) 2024-06-21T01:11:32.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 podman[115204]: 2024-06-21 01:11:32.282096742 +0000 UTC m=+0.365208078 container init 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81) 2024-06-21T01:11:32.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 podman[115204]: 2024-06-21 01:11:32.285456785 +0000 UTC m=+0.368568111 container start 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True) 2024-06-21T01:11:32.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 ceph-osd[115223]: -- 172.21.15.27:0/3017262779 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55faf8745a40 con 0x55faf9504000 2024-06-21T01:11:32.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 bash[115204]: 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 2024-06-21T01:11:32.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 systemd[1]: Started Ceph osd.3 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:11:33.193 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:11:32.832+0000 7ff29a37d640 -1 Falling back to public interface 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[84934]: pgmap v139: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[90235]: pgmap v139: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:33 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:33 smithi120 ceph-mon[67716]: pgmap v139: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:33.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:33 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:35 smithi120 ceph-mon[67716]: pgmap v140: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[84934]: pgmap v140: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:35.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[90235]: pgmap v140: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:35.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:35.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:35.729 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:11:35.554+0000 7ff29a37d640 -1 osd.3 0 read_superblock omap replica is missing. 2024-06-21T01:11:35.729 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:35 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:11:35.584+0000 7ff29a37d640 -1 osd.3 128 log_to_monitors true 2024-06-21T01:11:36.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:36 smithi027 ceph-mon[84934]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T01:11:36.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:36 smithi027 ceph-mon[90235]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T01:11:36.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:36 smithi120 ceph-mon[67716]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-21T01:11:37.228 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:11:36 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:11:36.829+0000 7ff28e120640 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: osdmap e131: 8 total, 7 up, 8 in 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:37 smithi120 ceph-mon[67716]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' 2024-06-21T01:11:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T01:11:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: osdmap e131: 8 total, 7 up, 8 in 2024-06-21T01:11:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[84934]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: pgmap v141: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 154 MiB used, 715 GiB / 715 GiB avail; 93/723 objects degraded (12.863%) 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: osdmap e131: 8 total, 7 up, 8 in 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:37 smithi027 ceph-mon[90235]: from='osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583]' entity='osd.3' 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 24 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583] boot 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: osdmap e132: 8 total, 8 up, 8 in 2024-06-21T01:11:38.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:38 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:11:38.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 24 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583] boot 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: osdmap e132: 8 total, 8 up, 8 in 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 93/723 objects degraded (12.863%), 24 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: osd.3 [v2:172.21.15.27:6826/3844250583,v1:172.21.15.27:6827/3844250583] boot 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: osdmap e132: 8 total, 8 up, 8 in 2024-06-21T01:11:38.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-21T01:11:38.922 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:38] "GET /metrics HTTP/1.1" 200 38166 "" "Prometheus/2.43.0" 2024-06-21T01:11:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:39 smithi120 ceph-mon[67716]: pgmap v144: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 93/723 objects degraded (12.863%) 2024-06-21T01:11:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:39 smithi120 ceph-mon[67716]: osdmap e133: 8 total, 8 up, 8 in 2024-06-21T01:11:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[84934]: pgmap v144: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 93/723 objects degraded (12.863%) 2024-06-21T01:11:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[84934]: osdmap e133: 8 total, 8 up, 8 in 2024-06-21T01:11:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[90235]: pgmap v144: 161 pgs: 42 active+undersized, 24 active+undersized+degraded, 95 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 93/723 objects degraded (12.863%) 2024-06-21T01:11:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:39 smithi027 ceph-mon[90235]: osdmap e133: 8 total, 8 up, 8 in 2024-06-21T01:11:40.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:40 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:11:40.801+0000 7f3f47d87640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-21T01:11:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: pgmap v146: 161 pgs: 22 active+undersized, 13 active+undersized+degraded, 126 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/723 objects degraded (7.884%) 2024-06-21T01:11:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[84934]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: pgmap v146: 161 pgs: 22 active+undersized, 13 active+undersized+degraded, 126 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/723 objects degraded (7.884%) 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:41.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:41.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:41.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:41 smithi027 ceph-mon[90235]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-21T01:11:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: pgmap v146: 161 pgs: 22 active+undersized, 13 active+undersized+degraded, 126 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 57/723 objects degraded (7.884%) 2024-06-21T01:11:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:41 smithi120 ceph-mon[67716]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-21T01:11:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:42 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:42.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:42 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:42 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:11:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[90235]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded) 2024-06-21T01:11:43.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:11:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[84934]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded) 2024-06-21T01:11:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:43 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:11:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:43 smithi120 ceph-mon[67716]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:43 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/723 objects degraded (7.884%), 13 pgs degraded) 2024-06-21T01:11:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:43 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:11:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:45 smithi027 ceph-mon[84934]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:45 smithi027 ceph-mon[90235]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:45 smithi120 ceph-mon[67716]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:47.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:47 smithi027 ceph-mon[84934]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:47.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:47 smithi027 ceph-mon[90235]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:47 smithi120 ceph-mon[67716]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:48] "GET /metrics HTTP/1.1" 200 38166 "" "Prometheus/2.43.0" 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[84934]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:49.970 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[90235]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:49.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:49.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:49.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:49.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:49 smithi120 ceph-mon[67716]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:11:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:51 smithi027 ceph-mon[84934]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:51.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:51 smithi027 ceph-mon[90235]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:52.024 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:11:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:51 smithi120 ceph-mon[67716]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (6m) 16s ago 28m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (6m) 4m ago 28m 82.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (7m) 16s ago 27m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (7m) 4m ago 33m 436M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (17m) 16s ago 34m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (4m) 16s ago 35m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (5m) 4m ago 33m 36.2M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (4m) 16s ago 33m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (7m) 16s ago 29m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:11:52.701 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (7m) 4m ago 29m 15.7M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (2m) 16s ago 32m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (2m) 16s ago 32m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (77s) 16s ago 32m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (20s) 16s ago 31m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (31m) 4m ago 31m 56.6M 3248M 17.2.0 e1d6a67b021e 8bc1bdef7af5 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (30m) 4m ago 30m 53.2M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (30m) 4m ago 30m 52.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (30m) 4m ago 30m 55.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (6m) 4m ago 29m 65.1M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (27m) 16s ago 27m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:11:52.702 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (27m) 4m ago 27m 77.9M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:11:52.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:52 smithi027 ceph-mon[90235]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:52.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:52 smithi027 ceph-mon[84934]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:52 smithi120 ceph-mon[67716]: from='client.64378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.118 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:11:53.118 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:11:53.118 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:11:53.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 4 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 9 2024-06-21T01:11:53.119 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:11:53.120 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:11:53.467 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:11:53.467 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:11:53.467 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:11:53.467 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-21T01:11:53.468 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:11:53.468 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "4/8 daemons upgraded", 2024-06-21T01:11:53.468 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:11:53.468 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:11:53.468 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:11:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[84934]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:11:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[84934]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[84934]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2887074566' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:53.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[90235]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:11:53.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[90235]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[90235]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:53.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:53 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2887074566' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:53 smithi120 ceph-mon[67716]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-06-21T01:11:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:53 smithi120 ceph-mon[67716]: from='client.54450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:53 smithi120 ceph-mon[67716]: from='client.54456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:53 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2887074566' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:11:54.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:54 smithi027 ceph-mon[84934]: from='client.64402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:54.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:54 smithi027 ceph-mon[90235]: from='client.64402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:55.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:54 smithi120 ceph-mon[67716]: from='client.64402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:11:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:55 smithi027 ceph-mon[84934]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:55.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:55 smithi027 ceph-mon[90235]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:55 smithi120 ceph-mon[67716]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:56.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:56 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:56 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:56 smithi120 ceph-mon[67716]: Upgrade: osd.4 is safe to restart 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[84934]: Upgrade: osd.4 is safe to restart 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-21T01:11:56.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:56 smithi027 ceph-mon[90235]: Upgrade: osd.4 is safe to restart 2024-06-21T01:11:57.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:57 smithi120 ceph-mon[67716]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:57.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:57 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:57.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:57 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:11:57.874 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:57 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:57.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[84934]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[90235]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-21T01:11:57.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:57 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:11:58.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:58 smithi027 ceph-mon[84934]: Upgrade: Updating osd.4 2024-06-21T01:11:58.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:58 smithi027 ceph-mon[84934]: Deploying daemon osd.4 on smithi120 2024-06-21T01:11:58.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:58 smithi027 ceph-mon[90235]: Upgrade: Updating osd.4 2024-06-21T01:11:58.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:58 smithi027 ceph-mon[90235]: Deploying daemon osd.4 on smithi120 2024-06-21T01:11:58.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:11:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:11:58] "GET /metrics HTTP/1.1" 200 38171 "" "Prometheus/2.43.0" 2024-06-21T01:11:58.980 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:58 smithi120 ceph-mon[67716]: Upgrade: Updating osd.4 2024-06-21T01:11:58.980 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:58 smithi120 ceph-mon[67716]: Deploying daemon osd.4 on smithi120 2024-06-21T01:11:59.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:59 smithi027 ceph-mon[84934]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:59.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:11:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:11:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:59 smithi027 ceph-mon[90235]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:11:59.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:11:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:00.096 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:59 smithi120 ceph-mon[67716]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:00.097 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:11:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:00.732 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:00 smithi120 systemd[1]: Stopping Ceph osd.4 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:12:00.732 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[29386]: 2024-06-21T01:12:00.678+0000 7f3cc0f89700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:12:00.732 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[29386]: 2024-06-21T01:12:00.678+0000 7f3cc0f89700 -1 osd.4 133 *** Got signal Terminated *** 2024-06-21T01:12:00.732 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[29386]: 2024-06-21T01:12:00.678+0000 7f3cc0f89700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:12:01.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:00 smithi120 ceph-mon[67716]: osd.4 marked itself down and dead 2024-06-21T01:12:01.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:00 smithi027 ceph-mon[84934]: osd.4 marked itself down and dead 2024-06-21T01:12:01.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:00 smithi027 ceph-mon[90235]: osd.4 marked itself down and dead 2024-06-21T01:12:01.443 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:01 smithi120 podman[77968]: 2024-06-21 01:12:01.162701651 +0000 UTC m=+0.565831780 container died 8bc1bdef7af55fb7861d6abf8fcd7bf76e709bf66cae6f7d5c10eb48915a76e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_CLEAN=True, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-21T01:12:01.747 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:01 smithi120 podman[77968]: 2024-06-21 01:12:01.545116979 +0000 UTC m=+0.948247108 container cleanup 8bc1bdef7af55fb7861d6abf8fcd7bf76e709bf66cae6f7d5c10eb48915a76e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, maintainer=Guillaume Abrioux , release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-06-21T01:12:01.747 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:01 smithi120 bash[77968]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4 2024-06-21T01:12:02.089 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:01 smithi120 ceph-mon[67716]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:02.090 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:01 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:02.090 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:01 smithi120 ceph-mon[67716]: osdmap e134: 8 total, 7 up, 8 in 2024-06-21T01:12:02.090 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:01 smithi120 podman[77985]: 2024-06-21 01:12:01.771500516 +0000 UTC m=+0.602728917 container remove 8bc1bdef7af55fb7861d6abf8fcd7bf76e709bf66cae6f7d5c10eb48915a76e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, name=centos-stream) 2024-06-21T01:12:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[84934]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[84934]: osdmap e134: 8 total, 7 up, 8 in 2024-06-21T01:12:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[90235]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:02.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:01 smithi027 ceph-mon[90235]: osdmap e134: 8 total, 7 up, 8 in 2024-06-21T01:12:02.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:01.995198187 +0000 UTC m=+0.017305608 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:02.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:02.150457997 +0000 UTC m=+0.172565417 container create 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-06-21T01:12:02.650 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:02.392329372 +0000 UTC m=+0.414436800 container init 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:12:02.650 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:02.395972685 +0000 UTC m=+0.418080109 container start 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.vendor=CentOS) 2024-06-21T01:12:02.650 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:02.525268924 +0000 UTC m=+0.547376344 container attach 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=) 2024-06-21T01:12:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:02 smithi120 ceph-mon[67716]: osdmap e135: 8 total, 7 up, 8 in 2024-06-21T01:12:03.116 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:02 smithi120 podman[78071]: 2024-06-21 01:12:02.653240424 +0000 UTC m=+0.675347849 container died 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True) 2024-06-21T01:12:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:02 smithi027 ceph-mon[84934]: osdmap e135: 8 total, 7 up, 8 in 2024-06-21T01:12:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:02 smithi027 ceph-mon[90235]: osdmap e135: 8 total, 7 up, 8 in 2024-06-21T01:12:03.452 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 podman[78094]: 2024-06-21 01:12:03.237392537 +0000 UTC m=+0.577572438 container remove 4add3b504b226dfaca6e6621e05952a85e70d60678c4d41c36db2a7b1760e62b (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:12:03.452 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.4.service: Deactivated successfully. 2024-06-21T01:12:03.452 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 systemd[1]: Stopped Ceph osd.4 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:12:03.453 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.4.service: Consumed 16.789s CPU time. 2024-06-21T01:12:03.753 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 systemd[1]: Starting Ceph osd.4 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:12:03.753 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 podman[78197]: 2024-06-21 01:12:03.575380212 +0000 UTC m=+0.017264520 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:03.754 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 podman[78197]: 2024-06-21 01:12:03.69695286 +0000 UTC m=+0.138837150 container create cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-06-21T01:12:04.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:03 smithi120 ceph-mon[67716]: pgmap v159: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:04.055 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:03 smithi120 ceph-mon[67716]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:04.055 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 podman[78197]: 2024-06-21 01:12:03.966667745 +0000 UTC m=+0.408552049 container init cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:12:04.056 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:03 smithi120 podman[78197]: 2024-06-21 01:12:03.969966246 +0000 UTC m=+0.411850542 container start cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:12:04.056 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 podman[78197]: 2024-06-21 01:12:04.058090347 +0000 UTC m=+0.499974638 container attach cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:12:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:03 smithi027 ceph-mon[84934]: pgmap v159: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:03 smithi027 ceph-mon[84934]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:04.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:03 smithi027 ceph-mon[90235]: pgmap v159: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:04.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:03 smithi027 ceph-mon[90235]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:04.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-21T01:12:04.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-21T01:12:04.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate[78212]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 bash[78197]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-21T01:12:04.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 podman[78197]: 2024-06-21 01:12:04.384637298 +0000 UTC m=+0.826521589 container died cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=) 2024-06-21T01:12:05.298 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:04 smithi120 podman[78324]: 2024-06-21 01:12:04.936895191 +0000 UTC m=+0.546593237 container remove cef9156142595e8e24f7a8fa59a7c139fd2ad4962841cf1d84e06dbd40365cdf (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618) 2024-06-21T01:12:05.298 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 podman[78384]: 2024-06-21 01:12:05.053292789 +0000 UTC m=+0.016996164 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:05.298 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 podman[78384]: 2024-06-21 01:12:05.17956209 +0000 UTC m=+0.143265508 container create c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618) 2024-06-21T01:12:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:05 smithi120 ceph-mon[67716]: pgmap v160: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:05.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 podman[78384]: 2024-06-21 01:12:05.431775333 +0000 UTC m=+0.395478711 container init c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:12:05.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 podman[78384]: 2024-06-21 01:12:05.435670169 +0000 UTC m=+0.399373544 container start c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-06-21T01:12:05.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 ceph-osd[78403]: -- 172.21.15.120:0/193060141 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cb6c4bfa40 con 0x55cb6d27e000 2024-06-21T01:12:05.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 bash[78384]: c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d 2024-06-21T01:12:05.617 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 systemd[1]: Started Ceph osd.4 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[90235]: pgmap v160: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:05.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:05 smithi027 ceph-mon[84934]: pgmap v160: 161 pgs: 19 peering, 15 stale+active+clean, 127 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:12:06.343 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:12:05.983+0000 7f177d75d640 -1 Falling back to public interface 2024-06-21T01:12:06.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:06 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:06.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:06.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:06 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:07 smithi120 ceph-mon[67716]: pgmap v161: 161 pgs: 31 active+undersized, 19 peering, 18 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-06-21T01:12:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:07 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 90/723 objects degraded (12.448%), 18 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:07 smithi027 ceph-mon[84934]: pgmap v161: 161 pgs: 31 active+undersized, 19 peering, 18 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-06-21T01:12:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:07 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 90/723 objects degraded (12.448%), 18 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:07.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:07 smithi027 ceph-mon[90235]: pgmap v161: 161 pgs: 31 active+undersized, 19 peering, 18 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-06-21T01:12:07.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:07 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 90/723 objects degraded (12.448%), 18 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:08.707 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:08 smithi120 ceph-mon[67716]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-21T01:12:08.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:08 smithi027 ceph-mon[84934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-21T01:12:08.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:08 smithi027 ceph-mon[90235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-21T01:12:08.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:08] "GET /metrics HTTP/1.1" 200 38171 "" "Prometheus/2.43.0" 2024-06-21T01:12:09.027 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:12:08.709+0000 7f177d75d640 -1 osd.4 0 read_superblock omap replica is missing. 2024-06-21T01:12:09.027 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:12:08.739+0000 7f177d75d640 -1 osd.4 133 log_to_monitors true 2024-06-21T01:12:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: pgmap v162: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 125/723 objects degraded (17.289%) 2024-06-21T01:12:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: pgmap v162: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 125/723 objects degraded (17.289%) 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:09.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:09 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.116 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:12:09.679+0000 7f1775508640 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: pgmap v162: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 125/723 objects degraded (17.289%) 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:09 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.799 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T01:12:10.800 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.800 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: osdmap e136: 8 total, 7 up, 8 in 2024-06-21T01:12:10.800 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.800 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.800 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:10 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T01:12:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: osdmap e136: 8 total, 7 up, 8 in 2024-06-21T01:12:10.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: from='osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: osdmap e136: 8 total, 7 up, 8 in 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:10.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:10 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: pgmap v164: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 125/723 objects degraded (17.289%) 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355] boot 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: osdmap e137: 8 total, 8 up, 8 in 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:12:11.823 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:11 smithi120 ceph-mon[67716]: osdmap e138: 8 total, 8 up, 8 in 2024-06-21T01:12:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: pgmap v164: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 125/723 objects degraded (17.289%) 2024-06-21T01:12:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355] boot 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: osdmap e137: 8 total, 8 up, 8 in 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[84934]: osdmap e138: 8 total, 8 up, 8 in 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: pgmap v164: 161 pgs: 42 active+undersized, 26 active+undersized+degraded, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 125/723 objects degraded (17.289%) 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: osd.4 [v2:172.21.15.120:6800/3145739355,v1:172.21.15.120:6801/3145739355] boot 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: osdmap e137: 8 total, 8 up, 8 in 2024-06-21T01:12:11.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-21T01:12:11.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:11 smithi027 ceph-mon[90235]: osdmap e138: 8 total, 8 up, 8 in 2024-06-21T01:12:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:12 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 125/723 objects degraded (17.289%), 26 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:12.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:12 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 125/723 objects degraded (17.289%), 26 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:12.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:12 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 125/723 objects degraded (17.289%), 26 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:12:13.440+0000 7f3f47d87640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: pgmap v167: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: pgmap v167: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:13.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:13.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:13 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: pgmap v167: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:14.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:13 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:14.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:14 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:14.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:14 smithi027 ceph-mon[90235]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-06-21T01:12:14.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:14 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:14.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:14 smithi027 ceph-mon[84934]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-06-21T01:12:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:14 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:14 smithi120 ceph-mon[67716]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-06-21T01:12:15.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:15 smithi027 ceph-mon[84934]: pgmap v168: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:15 smithi027 ceph-mon[90235]: pgmap v168: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:15 smithi120 ceph-mon[67716]: pgmap v168: 161 pgs: 11 peering, 36 active+undersized, 21 active+undersized+degraded, 93 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 101/723 objects degraded (13.970%) 2024-06-21T01:12:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:16 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded) 2024-06-21T01:12:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:16 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:12:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:16 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded) 2024-06-21T01:12:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:16 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:12:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:16 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 101/723 objects degraded (13.970%), 21 pgs degraded) 2024-06-21T01:12:17.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:16 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:12:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:17 smithi120 ceph-mon[67716]: pgmap v169: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-06-21T01:12:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:17 smithi027 ceph-mon[84934]: pgmap v169: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-06-21T01:12:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:17 smithi027 ceph-mon[90235]: pgmap v169: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-06-21T01:12:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:18] "GET /metrics HTTP/1.1" 200 38171 "" "Prometheus/2.43.0" 2024-06-21T01:12:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:19 smithi120 ceph-mon[67716]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:20.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[84934]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:20.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[90235]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:21 smithi120 ceph-mon[67716]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:12:22.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:21 smithi027 ceph-mon[84934]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:12:22.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:21 smithi027 ceph-mon[90235]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:12:23.843 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:12:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:23 smithi120 ceph-mon[67716]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 957 B/s rd, 0 op/s 2024-06-21T01:12:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:23 smithi027 ceph-mon[84934]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 957 B/s rd, 0 op/s 2024-06-21T01:12:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:23 smithi027 ceph-mon[90235]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 957 B/s rd, 0 op/s 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 47s ago 29m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (6m) 15s ago 29m 78.9M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (7m) 47s ago 28m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (8m) 15s ago 33m 441M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (18m) 47s ago 35m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:12:24.524 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (4m) 47s ago 35m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (5m) 15s ago 34m 45.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (4m) 47s ago 34m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (7m) 47s ago 29m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (7m) 15s ago 29m 16.1M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (3m) 47s ago 33m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (2m) 47s ago 33m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (109s) 47s ago 32m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (52s) 47s ago 32m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (19s) 15s ago 31m 11.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (31m) 15s ago 31m 57.5M 3248M 17.2.0 e1d6a67b021e dfd3834923ad 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (31m) 15s ago 31m 56.0M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (30m) 15s ago 30m 59.4M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (7m) 15s ago 30m 71.6M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (28m) 47s ago 28m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:12:24.525 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (28m) 15s ago 28m 79.7M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:12:24.937 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:12:24.937 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 5 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:12:24.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:24.939 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:12:24.939 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-21T01:12:24.939 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 10 2024-06-21T01:12:24.939 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:12:24.939 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:12:24.939 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:24 smithi027 ceph-mon[84934]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:24.939 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:24 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:24.940 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:24 smithi027 ceph-mon[90235]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:24.940 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:24 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:24 smithi120 ceph-mon[67716]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:25.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:24 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "5/8 daemons upgraded", 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:12:25.285 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:12:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:25 smithi120 ceph-mon[67716]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:25 smithi120 ceph-mon[67716]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:25 smithi120 ceph-mon[67716]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:25 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2237536880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[84934]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[84934]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[84934]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2237536880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[90235]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[90235]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[90235]: from='client.54489 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:26.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:25 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2237536880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:27.032 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:26 smithi120 ceph-mon[67716]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:27.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:26 smithi027 ceph-mon[84934]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:27.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:26 smithi027 ceph-mon[90235]: from='client.54501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:28.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:27 smithi120 ceph-mon[67716]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:27 smithi027 ceph-mon[84934]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:28.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:27 smithi027 ceph-mon[90235]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:28.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:28 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:28.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:28 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:28] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-06-21T01:12:29.079 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:28 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:29 smithi120 ceph-mon[67716]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:29 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:29 smithi120 ceph-mon[67716]: Upgrade: osd.5 is safe to restart 2024-06-21T01:12:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[84934]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[84934]: Upgrade: osd.5 is safe to restart 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[90235]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[90235]: Upgrade: osd.5 is safe to restart 2024-06-21T01:12:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[84934]: Upgrade: Updating osd.5 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[84934]: Deploying daemon osd.5 on smithi120 2024-06-21T01:12:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[90235]: Upgrade: Updating osd.5 2024-06-21T01:12:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:12:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:30 smithi027 ceph-mon[90235]: Deploying daemon osd.5 on smithi120 2024-06-21T01:12:31.290 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:30 smithi120 ceph-mon[67716]: Upgrade: Updating osd.5 2024-06-21T01:12:31.290 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:31.290 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-21T01:12:31.290 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:31.290 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:30 smithi120 ceph-mon[67716]: Deploying daemon osd.5 on smithi120 2024-06-21T01:12:32.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:31 smithi027 ceph-mon[84934]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:31 smithi027 ceph-mon[90235]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:32.289 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:31 smithi120 ceph-mon[67716]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:12:33.116 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:32 smithi120 systemd[1]: Stopping Ceph osd.5 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:12:33.616 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[32250]: 2024-06-21T01:12:33.128+0000 7fd331401700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:12:33.616 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[32250]: 2024-06-21T01:12:33.128+0000 7fd331401700 -1 osd.5 138 *** Got signal Terminated *** 2024-06-21T01:12:33.616 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[32250]: 2024-06-21T01:12:33.128+0000 7fd331401700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:12:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:33 smithi027 ceph-mon[84934]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:33 smithi027 ceph-mon[84934]: osd.5 marked itself down and dead 2024-06-21T01:12:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:33 smithi027 ceph-mon[90235]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:33 smithi027 ceph-mon[90235]: osd.5 marked itself down and dead 2024-06-21T01:12:34.332 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:33 smithi120 ceph-mon[67716]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:12:34.332 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:33 smithi120 ceph-mon[67716]: osd.5 marked itself down and dead 2024-06-21T01:12:34.616 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:34 smithi120 podman[82574]: 2024-06-21 01:12:34.335048998 +0000 UTC m=+1.306060622 container died dfd3834923ad3dbdbb632372f1751d445ee2e2f1ade6b7531406aaef5fa0d08c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-06-21T01:12:34.939 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:34 smithi120 podman[82574]: 2024-06-21 01:12:34.712739402 +0000 UTC m=+1.683751018 container cleanup dfd3834923ad3dbdbb632372f1751d445ee2e2f1ade6b7531406aaef5fa0d08c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.buildah.version=1.19.8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, name=centos-stream, distribution-scope=public, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-06-21T01:12:34.940 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:34 smithi120 bash[82574]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5 2024-06-21T01:12:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: osdmap e139: 8 total, 7 up, 8 in 2024-06-21T01:12:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]: dispatch 2024-06-21T01:12:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-06-21T01:12:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: osdmap e139: 8 total, 7 up, 8 in 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-06-21T01:12:35.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]: dispatch 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:35.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:35.337 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82589]: 2024-06-21 01:12:35.006188287 +0000 UTC m=+0.662896107 container remove dfd3834923ad3dbdbb632372f1751d445ee2e2f1ade6b7531406aaef5fa0d08c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, ceph=True, RELEASE=HEAD, version=8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream) 2024-06-21T01:12:35.337 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:12:35.337 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: osdmap e139: 8 total, 7 up, 8 in 2024-06-21T01:12:35.337 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]: dispatch 2024-06-21T01:12:35.337 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-06-21T01:12:35.337 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]: dispatch 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:35.338 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:35.615 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.242683906 +0000 UTC m=+0.017492621 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:35.615 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.348630088 +0000 UTC m=+0.123438795 container create 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240618) 2024-06-21T01:12:35.615 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.54288726 +0000 UTC m=+0.317695973 container init 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux ) 2024-06-21T01:12:35.615 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.546645337 +0000 UTC m=+0.321454043 container start 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-06-21T01:12:35.615 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.617588559 +0000 UTC m=+0.392397264 container attach 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-21T01:12:35.866 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:35 smithi120 podman[82679]: 2024-06-21 01:12:35.801579155 +0000 UTC m=+0.576387877 container died 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, RELEASE=squid-release-c65be81, ceph=True, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:12:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: pgmap v179: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:12:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]': finished 2024-06-21T01:12:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[84934]: osdmap e140: 8 total, 7 up, 8 in 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: pgmap v179: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-06-21T01:12:36.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]': finished 2024-06-21T01:12:36.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:35 smithi027 ceph-mon[90235]: osdmap e140: 8 total, 7 up, 8 in 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: pgmap v179: 161 pgs: 22 stale+active+clean, 139 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2, 3, 5]}]': finished 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.0", "id": [1, 5]}]': finished 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.5", "id": [0, 5]}]': finished 2024-06-21T01:12:36.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.6", "id": [7, 3]}]': finished 2024-06-21T01:12:36.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [4, 6]}]': finished 2024-06-21T01:12:36.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1d", "id": [1, 5, 4, 6]}]': finished 2024-06-21T01:12:36.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [4, 5, 7, 2]}]': finished 2024-06-21T01:12:36.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:35 smithi120 ceph-mon[67716]: osdmap e140: 8 total, 7 up, 8 in 2024-06-21T01:12:36.853 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 podman[82702]: 2024-06-21 01:12:36.438330521 +0000 UTC m=+0.631073040 container remove 0d810074e54ff32f77600e550a18884d47647c8ab6a03a91144ec687de64f10f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:12:36.853 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.5.service: Deactivated successfully. 2024-06-21T01:12:36.854 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 systemd[1]: Stopped Ceph osd.5 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:12:36.854 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.5.service: Consumed 16.898s CPU time. 2024-06-21T01:12:36.854 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 systemd[1]: Starting Ceph osd.5 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:12:37.114 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:36 smithi120 ceph-mon[67716]: osdmap e141: 8 total, 7 up, 8 in 2024-06-21T01:12:37.114 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 podman[82803]: 2024-06-21 01:12:36.85621635 +0000 UTC m=+0.099084603 container create b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:12:37.115 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:36 smithi120 podman[82803]: 2024-06-21 01:12:36.774282105 +0000 UTC m=+0.017150364 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:37.115 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 podman[82803]: 2024-06-21 01:12:37.117459811 +0000 UTC m=+0.360328076 container init b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, RELEASE=squid-release-c65be81) 2024-06-21T01:12:37.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:36 smithi027 ceph-mon[84934]: osdmap e141: 8 total, 7 up, 8 in 2024-06-21T01:12:37.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:36 smithi027 ceph-mon[90235]: osdmap e141: 8 total, 7 up, 8 in 2024-06-21T01:12:37.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 podman[82803]: 2024-06-21 01:12:37.121028664 +0000 UTC m=+0.363896918 container start b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-21T01:12:37.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 podman[82803]: 2024-06-21 01:12:37.208985818 +0000 UTC m=+0.451854080 container attach b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81) 2024-06-21T01:12:37.857 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-21T01:12:37.858 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate[82820]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-21T01:12:37.859 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 bash[82803]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-21T01:12:37.859 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:37 smithi120 podman[82803]: 2024-06-21 01:12:37.533466808 +0000 UTC m=+0.776335062 container died b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-21T01:12:38.116 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 podman[82938]: 2024-06-21 01:12:38.07113098 +0000 UTC m=+0.528367749 container remove b24a93538a9f37999f0c5a297ba924ac7d9d96da8036d6bcfccae675c101abdd (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0) 2024-06-21T01:12:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-mon[67716]: pgmap v182: 161 pgs: 1 unknown, 23 active+undersized, 12 peering, 6 stale+active+clean, 10 active+undersized+degraded, 109 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 33/723 objects degraded (4.564%) 2024-06-21T01:12:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-mon[67716]: osdmap e142: 8 total, 7 up, 8 in 2024-06-21T01:12:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-mon[67716]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:37 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 10 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[84934]: pgmap v182: 161 pgs: 1 unknown, 23 active+undersized, 12 peering, 6 stale+active+clean, 10 active+undersized+degraded, 109 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 33/723 objects degraded (4.564%) 2024-06-21T01:12:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[84934]: osdmap e142: 8 total, 7 up, 8 in 2024-06-21T01:12:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[84934]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 10 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[90235]: pgmap v182: 161 pgs: 1 unknown, 23 active+undersized, 12 peering, 6 stale+active+clean, 10 active+undersized+degraded, 109 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 33/723 objects degraded (4.564%) 2024-06-21T01:12:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[90235]: osdmap e142: 8 total, 7 up, 8 in 2024-06-21T01:12:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[90235]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-06-21T01:12:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:37 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 33/723 objects degraded (4.564%), 10 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:38.532 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 podman[82999]: 2024-06-21 01:12:38.185669305 +0000 UTC m=+0.017605429 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:12:38.532 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 podman[82999]: 2024-06-21 01:12:38.31351553 +0000 UTC m=+0.145451642 container create a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-06-21T01:12:38.532 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 podman[82999]: 2024-06-21 01:12:38.532332007 +0000 UTC m=+0.364268120 container init a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-21T01:12:38.532 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 podman[82999]: 2024-06-21 01:12:38.535488852 +0000 UTC m=+0.367424968 container start a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-06-21T01:12:38.866 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 ceph-osd[83018]: -- 172.21.15.120:0/4066787135 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558aff6c1a40 con 0x558aff69fc00 2024-06-21T01:12:38.866 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 bash[82999]: a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 2024-06-21T01:12:38.866 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:38 smithi120 systemd[1]: Started Ceph osd.5 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:12:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:38] "GET /metrics HTTP/1.1" 200 38174 "" "Prometheus/2.43.0" 2024-06-21T01:12:39.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:12:39.078+0000 7f51f7318640 -1 Falling back to public interface 2024-06-21T01:12:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-mon[67716]: pgmap v184: 161 pgs: 1 remapped+peering, 1 unknown, 31 active+undersized, 12 peering, 18 active+undersized+degraded, 98 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-06-21T01:12:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:39.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[84934]: pgmap v184: 161 pgs: 1 remapped+peering, 1 unknown, 31 active+undersized, 12 peering, 18 active+undersized+degraded, 98 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[90235]: pgmap v184: 161 pgs: 1 remapped+peering, 1 unknown, 31 active+undersized, 12 peering, 18 active+undersized+degraded, 98 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 65/723 objects degraded (8.990%) 2024-06-21T01:12:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:39.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:41.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:40 smithi120 ceph-mon[67716]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-06-21T01:12:41.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:40 smithi027 ceph-mon[84934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-06-21T01:12:41.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:40 smithi027 ceph-mon[90235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-06-21T01:12:42.000 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:41 smithi120 ceph-mon[67716]: pgmap v185: 161 pgs: 5 active+clean+remapped, 1 remapped+peering, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:42.001 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:12:41.777+0000 7f51f7318640 -1 osd.5 0 read_superblock omap replica is missing. 2024-06-21T01:12:42.001 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:41 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:12:41.802+0000 7f51f7318640 -1 osd.5 138 log_to_monitors true 2024-06-21T01:12:42.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:41 smithi027 ceph-mon[84934]: pgmap v185: 161 pgs: 5 active+clean+remapped, 1 remapped+peering, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:42.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:41 smithi027 ceph-mon[90235]: pgmap v185: 161 pgs: 5 active+clean+remapped, 1 remapped+peering, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:42 smithi120 ceph-mon[67716]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:42 smithi120 ceph-mon[67716]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:42 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:43.116 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:12:42 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:12:42.768+0000 7f51ef0c3640 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[84934]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[84934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[90235]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[90235]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-21T01:12:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:42 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 81/723 objects degraded (11.203%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:44.006 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: pgmap v186: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: osdmap e143: 8 total, 7 up, 8 in 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.007 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:43 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: pgmap v186: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: osdmap e143: 8 total, 7 up, 8 in 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: pgmap v186: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 191 MiB used, 715 GiB / 715 GiB avail; 81/723 objects degraded (11.203%) 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: osdmap e143: 8 total, 7 up, 8 in 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:44.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:43 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:45.015 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:44 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:45.015 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:44 smithi120 ceph-mon[67716]: osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867] boot 2024-06-21T01:12:45.016 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:44 smithi120 ceph-mon[67716]: osdmap e144: 8 total, 8 up, 8 in 2024-06-21T01:12:45.016 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:44 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:12:45.016 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:44 smithi120 ceph-mon[67716]: osdmap e145: 8 total, 8 up, 8 in 2024-06-21T01:12:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[84934]: osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867] boot 2024-06-21T01:12:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[84934]: osdmap e144: 8 total, 8 up, 8 in 2024-06-21T01:12:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:12:45.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[84934]: osdmap e145: 8 total, 8 up, 8 in 2024-06-21T01:12:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:12:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[90235]: osd.5 [v2:172.21.15.120:6808/801069867,v1:172.21.15.120:6809/801069867] boot 2024-06-21T01:12:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[90235]: osdmap e144: 8 total, 8 up, 8 in 2024-06-21T01:12:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-21T01:12:45.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:44 smithi027 ceph-mon[90235]: osdmap e145: 8 total, 8 up, 8 in 2024-06-21T01:12:46.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:45 smithi120 ceph-mon[67716]: pgmap v189: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 691 B/s rd, 0 op/s; 81/723 objects degraded (11.203%) 2024-06-21T01:12:46.085 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:45 smithi120 ceph-mon[67716]: osdmap e146: 8 total, 8 up, 8 in 2024-06-21T01:12:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:45 smithi027 ceph-mon[84934]: pgmap v189: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 691 B/s rd, 0 op/s; 81/723 objects degraded (11.203%) 2024-06-21T01:12:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:45 smithi027 ceph-mon[84934]: osdmap e146: 8 total, 8 up, 8 in 2024-06-21T01:12:46.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:45 smithi027 ceph-mon[90235]: pgmap v189: 161 pgs: 6 active+clean+remapped, 36 active+undersized, 22 active+undersized+degraded, 97 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 691 B/s rd, 0 op/s; 81/723 objects degraded (11.203%) 2024-06-21T01:12:46.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:45 smithi027 ceph-mon[90235]: osdmap e146: 8 total, 8 up, 8 in 2024-06-21T01:12:47.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:46 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:12:46.869+0000 7f3f47d87640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: pgmap v192: 161 pgs: 7 peering, 4 active+clean+remapped, 17 active+undersized, 15 active+undersized+degraded, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-21T01:12:48.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:47 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: pgmap v192: 161 pgs: 7 peering, 4 active+clean+remapped, 17 active+undersized, 15 active+undersized+degraded, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: pgmap v192: 161 pgs: 7 peering, 4 active+clean+remapped, 17 active+undersized, 15 active+undersized+degraded, 118 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:12:48.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-06-21T01:12:48.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:47 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded (PG_DEGRADED) 2024-06-21T01:12:48.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:48 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded) 2024-06-21T01:12:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:48 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:12:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:48] "GET /metrics HTTP/1.1" 200 38174 "" "Prometheus/2.43.0" 2024-06-21T01:12:48.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:48 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded) 2024-06-21T01:12:48.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:48 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:12:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:48 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/723 objects degraded (7.746%), 15 pgs degraded) 2024-06-21T01:12:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:48 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:12:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:49 smithi120 ceph-mon[67716]: pgmap v193: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-06-21T01:12:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[84934]: pgmap v193: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-06-21T01:12:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:50.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[90235]: pgmap v193: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-06-21T01:12:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:12:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:12:50.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:12:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:51 smithi120 ceph-mon[67716]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-06-21T01:12:52.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:51 smithi027 ceph-mon[84934]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-06-21T01:12:52.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:51 smithi027 ceph-mon[90235]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 op/s 2024-06-21T01:12:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:53 smithi120 ceph-mon[67716]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-06-21T01:12:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:53 smithi027 ceph-mon[84934]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-06-21T01:12:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:53 smithi027 ceph-mon[90235]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-06-21T01:12:55.656 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:12:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:55 smithi027 ceph-mon[84934]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:12:55.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:55 smithi027 ceph-mon[90235]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:12:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:55 smithi120 ceph-mon[67716]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 79s ago 29m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (7m) 13s ago 29m 82.0M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (8m) 79s ago 28m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (8m) 13s ago 34m 441M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (18m) 79s ago 35m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (5m) 79s ago 36m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (6m) 13s ago 34m 47.3M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:12:56.334 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 79s ago 34m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 79s ago 30m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (8m) 13s ago 30m 16.2M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (3m) 79s ago 33m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (3m) 79s ago 33m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (2m) 79s ago 33m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (84s) 79s ago 32m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (51s) 13s ago 32m 44.9M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (18s) 13s ago 32m 13.5M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (31m) 13s ago 31m 56.9M 3248M 17.2.0 e1d6a67b021e 66a44302d5c6 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (31m) 13s ago 31m 60.2M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (7m) 13s ago 30m 71.6M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (29m) 79s ago 28m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:12:56.335 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (28m) 13s ago 28m 79.8M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:12:56.752 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:12:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 6 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:12:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:12:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-21T01:12:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 11 2024-06-21T01:12:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:12:56.754 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:12:57.099 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:12:57.099 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "6/8 daemons upgraded", 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:12:57.100 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:12:57.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[84934]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[84934]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/213224335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:57.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[90235]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[90235]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:56 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/213224335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:56 smithi120 ceph-mon[67716]: from='client.54513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:56 smithi120 ceph-mon[67716]: from='client.54519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:57.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:56 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/213224335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:12:58.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:57 smithi120 ceph-mon[67716]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:57 smithi120 ceph-mon[67716]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:12:58.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:57 smithi120 ceph-mon[67716]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[90235]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[90235]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[90235]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[84934]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[84934]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:12:58.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:57 smithi027 ceph-mon[84934]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:12:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:12:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:12:58] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-06-21T01:13:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:59 smithi120 ceph-mon[67716]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:13:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:00.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:12:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[84934]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:13:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:00.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[90235]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-06-21T01:13:00.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:00.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:12:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:01 smithi120 ceph-mon[67716]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:02.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:01 smithi027 ceph-mon[84934]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:02.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:01 smithi027 ceph-mon[90235]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:02 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:02 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:02 smithi120 ceph-mon[67716]: Upgrade: osd.6 is safe to restart 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[84934]: Upgrade: osd.6 is safe to restart 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-21T01:13:03.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:02 smithi027 ceph-mon[90235]: Upgrade: osd.6 is safe to restart 2024-06-21T01:13:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:03 smithi120 ceph-mon[67716]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:13:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:03 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[84934]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:13:04.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:04.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[90235]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:04.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:04.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-21T01:13:04.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:03 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:04.969 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:04 smithi120 ceph-mon[67716]: Upgrade: Updating osd.6 2024-06-21T01:13:04.969 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:04 smithi120 ceph-mon[67716]: Deploying daemon osd.6 on smithi120 2024-06-21T01:13:04.969 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[84934]: Upgrade: Updating osd.6 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[84934]: Deploying daemon osd.6 on smithi120 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[90235]: Upgrade: Updating osd.6 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[90235]: Deploying daemon osd.6 on smithi120 2024-06-21T01:13:05.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:06.060 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:05 smithi120 ceph-mon[67716]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:06.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:05 smithi027 ceph-mon[84934]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:06.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:05 smithi027 ceph-mon[90235]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:06.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:06 smithi120 systemd[1]: Stopping Ceph osd.6 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:13:06.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[35130]: 2024-06-21T01:13:06.305+0000 7fde7bd74700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:13:06.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[35130]: 2024-06-21T01:13:06.305+0000 7fde7bd74700 -1 osd.6 146 *** Got signal Terminated *** 2024-06-21T01:13:06.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[35130]: 2024-06-21T01:13:06.305+0000 7fde7bd74700 -1 osd.6 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:13:07.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:06 smithi120 ceph-mon[67716]: osd.6 marked itself down and dead 2024-06-21T01:13:07.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:06 smithi027 ceph-mon[84934]: osd.6 marked itself down and dead 2024-06-21T01:13:07.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:06 smithi027 ceph-mon[90235]: osd.6 marked itself down and dead 2024-06-21T01:13:07.687 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:07 smithi120 podman[87214]: 2024-06-21 01:13:07.410107357 +0000 UTC m=+1.200562160 container died 66a44302d5c6f0e48e008db612437198a51f38891c20311ca0cc992b230f8d1a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, com.redhat.component=centos-stream-container, name=centos-stream, version=8, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-21T01:13:08.104 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:07 smithi120 ceph-mon[67716]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:08.104 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:07 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:08.104 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:07 smithi120 ceph-mon[67716]: osdmap e147: 8 total, 7 up, 8 in 2024-06-21T01:13:08.105 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:07 smithi120 podman[87214]: 2024-06-21 01:13:07.78053774 +0000 UTC m=+1.570992545 container cleanup 66a44302d5c6f0e48e008db612437198a51f38891c20311ca0cc992b230f8d1a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, vcs-type=git, release=754, distribution-scope=public, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-06-21T01:13:08.105 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:07 smithi120 bash[87214]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6 2024-06-21T01:13:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[84934]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:08.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[84934]: osdmap e147: 8 total, 7 up, 8 in 2024-06-21T01:13:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[90235]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:08.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:08.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:07 smithi027 ceph-mon[90235]: osdmap e147: 8 total, 7 up, 8 in 2024-06-21T01:13:08.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87231]: 2024-06-21 01:13:08.107187697 +0000 UTC m=+0.697458920 container remove 66a44302d5c6f0e48e008db612437198a51f38891c20311ca0cc992b230f8d1a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, vendor=Red Hat, Inc., name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public) 2024-06-21T01:13:08.716 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.333526994 +0000 UTC m=+0.017891447 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:08.716 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.441297619 +0000 UTC m=+0.125662067 container create f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS) 2024-06-21T01:13:08.716 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.62721625 +0000 UTC m=+0.311580712 container init f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:13:08.716 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.630671287 +0000 UTC m=+0.315035737 container start f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD) 2024-06-21T01:13:08.716 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.718897792 +0000 UTC m=+0.403262253 container attach f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0) 2024-06-21T01:13:08.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:08 smithi027 ceph-mon[84934]: osdmap e148: 8 total, 7 up, 8 in 2024-06-21T01:13:08.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:08 smithi027 ceph-mon[90235]: osdmap e148: 8 total, 7 up, 8 in 2024-06-21T01:13:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:08] "GET /metrics HTTP/1.1" 200 38190 "" "Prometheus/2.43.0" 2024-06-21T01:13:09.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:08 smithi120 ceph-mon[67716]: osdmap e148: 8 total, 7 up, 8 in 2024-06-21T01:13:09.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:08 smithi120 podman[87316]: 2024-06-21 01:13:08.889480623 +0000 UTC m=+0.573845077 container died f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:13:09.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:09 smithi120 podman[87339]: 2024-06-21 01:13:09.57326823 +0000 UTC m=+0.678406585 container remove f4d8736d70f594c82c4dfc4ca60abbe2cd73a890cdbb86603fc6f9535791f582 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618) 2024-06-21T01:13:09.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:09 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.6.service: Deactivated successfully. 2024-06-21T01:13:09.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:09 smithi120 systemd[1]: Stopped Ceph osd.6 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:13:09.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:09 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.6.service: Consumed 14.764s CPU time. 2024-06-21T01:13:09.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:09 smithi120 systemd[1]: Starting Ceph osd.6 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:13:10.143 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:09 smithi120 ceph-mon[67716]: pgmap v205: 161 pgs: 13 peering, 10 stale+active+clean, 138 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:10.143 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:10.143 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:09.91849046 +0000 UTC m=+0.017333135 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:10.143 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:10.023680079 +0000 UTC m=+0.122522780 container create 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81) 2024-06-21T01:13:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:09 smithi027 ceph-mon[84934]: pgmap v205: 161 pgs: 13 peering, 10 stale+active+clean, 138 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:10.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:09 smithi027 ceph-mon[90235]: pgmap v205: 161 pgs: 13 peering, 10 stale+active+clean, 138 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:10.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:10.609 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:10.24728831 +0000 UTC m=+0.346130968 container init 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-06-21T01:13:10.609 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:10.250974743 +0000 UTC m=+0.349817405 container start 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, ceph=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-21T01:13:10.609 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:10.334722354 +0000 UTC m=+0.433565020 container attach 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81) 2024-06-21T01:13:10.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-21T01:13:10.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate[87454]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 bash[87439]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-21T01:13:10.867 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:10 smithi120 podman[87439]: 2024-06-21 01:13:10.666077705 +0000 UTC m=+0.764920364 container died 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81) 2024-06-21T01:13:11.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:10 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 55/723 objects degraded (7.607%), 9 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:11.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:10 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 55/723 objects degraded (7.607%), 9 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:11.235 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:10 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 55/723 objects degraded (7.607%), 9 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:11.592 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 podman[87567]: 2024-06-21 01:13:11.238245908 +0000 UTC m=+0.566562733 container remove 81cc4739ddbbfa853e2d01991c6079f6740ea81fc19b784156761f08e1edd928 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-activate, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618) 2024-06-21T01:13:11.592 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 podman[87630]: 2024-06-21 01:13:11.353511295 +0000 UTC m=+0.017466977 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:11.592 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 podman[87630]: 2024-06-21 01:13:11.464203183 +0000 UTC m=+0.128158858 container create cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-21T01:13:11.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 podman[87630]: 2024-06-21 01:13:11.691985313 +0000 UTC m=+0.355941003 container init cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD) 2024-06-21T01:13:11.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 podman[87630]: 2024-06-21 01:13:11.695173066 +0000 UTC m=+0.359128749 container start cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618) 2024-06-21T01:13:11.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 ceph-osd[87649]: -- 172.21.15.120:0/2725250741 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5574da137a40 con 0x5574daef6400 2024-06-21T01:13:11.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 bash[87630]: cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 2024-06-21T01:13:11.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:11 smithi120 systemd[1]: Started Ceph osd.6 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[84934]: pgmap v206: 161 pgs: 20 active+undersized, 13 peering, 3 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[90235]: pgmap v206: 161 pgs: 20 active+undersized, 13 peering, 3 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:11 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:12.229 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:11 smithi120 ceph-mon[67716]: pgmap v206: 161 pgs: 20 active+undersized, 13 peering, 3 stale+active+clean, 9 active+undersized+degraded, 116 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 55/723 objects degraded (7.607%) 2024-06-21T01:13:12.229 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.229 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:12.229 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:11 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:12.508 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:13:12.230+0000 7f9b0f676640 -1 Falling back to public interface 2024-06-21T01:13:14.148 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:13 smithi120 ceph-mon[67716]: pgmap v207: 161 pgs: 27 active+undersized, 13 peering, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-21T01:13:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:13 smithi027 ceph-mon[84934]: pgmap v207: 161 pgs: 27 active+undersized, 13 peering, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-21T01:13:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:13 smithi027 ceph-mon[90235]: pgmap v207: 161 pgs: 27 active+undersized, 13 peering, 15 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 72/723 objects degraded (9.959%) 2024-06-21T01:13:15.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:13:14.962+0000 7f9b0f676640 -1 osd.6 0 read_superblock omap replica is missing. 2024-06-21T01:13:15.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:13:14.986+0000 7f9b0f676640 -1 osd.6 146 log_to_monitors true 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[84934]: pgmap v208: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[84934]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[84934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[90235]: pgmap v208: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-06-21T01:13:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[90235]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[90235]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:15 smithi120 ceph-mon[67716]: pgmap v208: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 207 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 85/723 objects degraded (11.757%) 2024-06-21T01:13:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:15 smithi120 ceph-mon[67716]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:15 smithi120 ceph-mon[67716]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-21T01:13:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:16.866 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:13:16.413+0000 7f9b03419640 -1 osd.6 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:13:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T01:13:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: osdmap e149: 8 total, 7 up, 8 in 2024-06-21T01:13:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='osd.6 ' entity='osd.6' 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: osdmap e149: 8 total, 7 up, 8 in 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='osd.6 ' entity='osd.6' 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:17.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:17.285 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-21T01:13:17.285 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: osdmap e149: 8 total, 7 up, 8 in 2024-06-21T01:13:17.285 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.285 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:17.286 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='osd.6 ' entity='osd.6' 2024-06-21T01:13:17.286 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:17.286 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: pgmap v210: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843] boot 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: osdmap e150: 8 total, 8 up, 8 in 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:13:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:17 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: pgmap v210: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-21T01:13:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843] boot 2024-06-21T01:13:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: osdmap e150: 8 total, 8 up, 8 in 2024-06-21T01:13:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: pgmap v210: 161 pgs: 36 active+undersized, 19 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 85/723 objects degraded (11.757%) 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: osd.6 [v2:172.21.15.120:6816/3611908843,v1:172.21.15.120:6817/3611908843] boot 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: osdmap e150: 8 total, 8 up, 8 in 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-21T01:13:18.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:17 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 85/723 objects degraded (11.757%), 19 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:18] "GET /metrics HTTP/1.1" 200 38190 "" "Prometheus/2.43.0" 2024-06-21T01:13:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:19 smithi120 ceph-mon[67716]: pgmap v212: 161 pgs: 10 peering, 29 active+undersized, 16 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-06-21T01:13:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:19 smithi120 ceph-mon[67716]: osdmap e151: 8 total, 8 up, 8 in 2024-06-21T01:13:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:19 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:19.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:19.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[84934]: pgmap v212: 161 pgs: 10 peering, 29 active+undersized, 16 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-06-21T01:13:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[84934]: osdmap e151: 8 total, 8 up, 8 in 2024-06-21T01:13:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:19.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[90235]: pgmap v212: 161 pgs: 10 peering, 29 active+undersized, 16 active+undersized+degraded, 106 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-06-21T01:13:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[90235]: osdmap e151: 8 total, 8 up, 8 in 2024-06-21T01:13:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:19.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:20.228 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:19 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:13:19.880+0000 7f3f47d87640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:21.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:20 smithi120 ceph-mon[67716]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:13:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:21.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[84934]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:21.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:21.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:20 smithi027 ceph-mon[90235]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-06-21T01:13:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:21 smithi120 ceph-mon[67716]: pgmap v214: 161 pgs: 10 peering, 15 active+undersized, 8 active+undersized+degraded, 128 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-21T01:13:22.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:21 smithi027 ceph-mon[84934]: pgmap v214: 161 pgs: 10 peering, 15 active+undersized, 8 active+undersized+degraded, 128 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-21T01:13:22.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:21 smithi027 ceph-mon[90235]: pgmap v214: 161 pgs: 10 peering, 15 active+undersized, 8 active+undersized+degraded, 128 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/723 objects degraded (3.873%) 2024-06-21T01:13:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:22 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:22 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:22 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:23 smithi120 ceph-mon[67716]: pgmap v215: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:23 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded) 2024-06-21T01:13:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:23 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[90235]: pgmap v215: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded) 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[84934]: pgmap v215: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/723 objects degraded (3.873%), 8 pgs degraded) 2024-06-21T01:13:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:23 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:13:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:25 smithi120 ceph-mon[67716]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:13:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:25 smithi027 ceph-mon[84934]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:13:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:25 smithi027 ceph-mon[90235]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-06-21T01:13:27.475 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:13:28.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:27 smithi120 ceph-mon[67716]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 342 B/s rd, 0 op/s 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 111s ago 30m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (7m) 12s ago 30m 79.0M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (9m) 111s ago 29m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (9m) 12s ago 34m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (19m) 111s ago 36m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 111s ago 36m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (6m) 12s ago 35m 48.7M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 111s ago 35m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:13:28.149 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 111s ago 31m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (8m) 12s ago 30m 16.1M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (4m) 111s ago 34m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (3m) 111s ago 34m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (2m) 111s ago 33m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (116s) 111s ago 33m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (83s) 12s ago 32m 45.5M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (49s) 12s ago 32m 46.7M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (16s) 12s ago 32m 12.1M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (31m) 12s ago 31m 61.3M 3248M 17.2.0 e1d6a67b021e 50b38024b688 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (8m) 12s ago 31m 71.6M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (29m) 111s ago 29m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:13:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (29m) 12s ago 29m 80.0M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:13:28.151 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:27 smithi027 ceph-mon[84934]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 342 B/s rd, 0 op/s 2024-06-21T01:13:28.151 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:27 smithi027 ceph-mon[90235]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 342 B/s rd, 0 op/s 2024-06-21T01:13:28.564 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:13:28.564 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:13:28.564 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:13:28.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 7 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 12 2024-06-21T01:13:28.565 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:13:28.566 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:13:28.911 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "7/8 daemons upgraded", 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:13:28.912 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[90235]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[90235]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[90235]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/321247945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:28] "GET /metrics HTTP/1.1" 200 38055 "" "Prometheus/2.43.0" 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[84934]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[84934]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[84934]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:28.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:28 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/321247945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:29.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:28 smithi120 ceph-mon[67716]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:29.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:28 smithi120 ceph-mon[67716]: from='client.54549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:29.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:28 smithi120 ceph-mon[67716]: from='client.54555 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:29.073 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:28 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/321247945' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:29 smithi120 ceph-mon[67716]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-06-21T01:13:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:29 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:29 smithi120 ceph-mon[67716]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:29 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[84934]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[84934]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[90235]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-06-21T01:13:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[90235]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:13:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:29 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:31 smithi120 ceph-mon[67716]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-06-21T01:13:32.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:31 smithi027 ceph-mon[84934]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-06-21T01:13:32.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:31 smithi027 ceph-mon[90235]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 684 B/s rd, 0 op/s 2024-06-21T01:13:34.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:33 smithi120 ceph-mon[67716]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:13:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:33 smithi027 ceph-mon[84934]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:13:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:33 smithi027 ceph-mon[90235]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: Upgrade: osd.7 is safe to restart 2024-06-21T01:13:35.582 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]: dispatch 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: Upgrade: osd.7 is safe to restart 2024-06-21T01:13:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]: dispatch 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: Upgrade: osd.7 is safe to restart 2024-06-21T01:13:35.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]: dispatch 2024-06-21T01:13:36.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:36 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]': finished 2024-06-21T01:13:36.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:36 smithi120 ceph-mon[67716]: osdmap e152: 8 total, 8 up, 8 in 2024-06-21T01:13:36.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:36 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]': finished 2024-06-21T01:13:36.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:36 smithi027 ceph-mon[84934]: osdmap e152: 8 total, 8 up, 8 in 2024-06-21T01:13:36.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:36 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1f", "id": [7, 2]}]': finished 2024-06-21T01:13:36.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:36 smithi027 ceph-mon[90235]: osdmap e152: 8 total, 8 up, 8 in 2024-06-21T01:13:37.593 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: osdmap e153: 8 total, 8 up, 8 in 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: pgmap v224: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: Upgrade: Updating osd.7 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:37.594 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:37 smithi120 ceph-mon[67716]: Deploying daemon osd.7 on smithi120 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: osdmap e153: 8 total, 8 up, 8 in 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: pgmap v224: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: Upgrade: Updating osd.7 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:37.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[84934]: Deploying daemon osd.7 on smithi120 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: osdmap e153: 8 total, 8 up, 8 in 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: pgmap v224: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: Upgrade: Updating osd.7 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:37.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:37 smithi027 ceph-mon[90235]: Deploying daemon osd.7 on smithi120 2024-06-21T01:13:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:38] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-06-21T01:13:39.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:39 smithi027 ceph-mon[84934]: pgmap v225: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:39.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:39 smithi027 ceph-mon[90235]: pgmap v225: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:39.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:39.732 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:39 smithi120 ceph-mon[67716]: pgmap v225: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:13:39.732 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:39.732 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:39 smithi120 systemd[1]: Stopping Ceph osd.7 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:13:40.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[38096]: 2024-06-21T01:13:39.733+0000 7f3a289d2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:13:40.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[38096]: 2024-06-21T01:13:39.733+0000 7f3a289d2700 -1 osd.7 153 *** Got signal Terminated *** 2024-06-21T01:13:40.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:39 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[38096]: 2024-06-21T01:13:39.733+0000 7f3a289d2700 -1 osd.7 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:13:40.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:40 smithi120 ceph-mon[67716]: osd.7 marked itself down and dead 2024-06-21T01:13:40.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:40 smithi027 ceph-mon[84934]: osd.7 marked itself down and dead 2024-06-21T01:13:40.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:40 smithi027 ceph-mon[90235]: osd.7 marked itself down and dead 2024-06-21T01:13:41.083 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:40 smithi120 podman[92140]: 2024-06-21 01:13:40.803030856 +0000 UTC m=+1.191892722 container died 50b38024b6884fd9fab438f7f6a209ebaa82205c74f64fd22f327fdabb6f853d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, release=754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-06-21T01:13:41.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:41 smithi120 podman[92140]: 2024-06-21 01:13:41.158995395 +0000 UTC m=+1.547857257 container cleanup 50b38024b6884fd9fab438f7f6a209ebaa82205c74f64fd22f327fdabb6f853d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, name=centos-stream, ceph=True, GIT_CLEAN=True, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vendor=Red Hat, Inc.) 2024-06-21T01:13:41.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:41 smithi120 bash[92140]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7 2024-06-21T01:13:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[84934]: pgmap v226: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[84934]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[84934]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-21T01:13:41.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[84934]: osdmap e154: 8 total, 7 up, 8 in 2024-06-21T01:13:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[90235]: pgmap v226: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[90235]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[90235]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-21T01:13:41.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:41 smithi027 ceph-mon[90235]: osdmap e154: 8 total, 7 up, 8 in 2024-06-21T01:13:41.774 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:41 smithi120 ceph-mon[67716]: pgmap v226: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:13:41.774 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:41 smithi120 ceph-mon[67716]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-21T01:13:41.774 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:41 smithi120 ceph-mon[67716]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-21T01:13:41.774 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:41 smithi120 ceph-mon[67716]: osdmap e154: 8 total, 7 up, 8 in 2024-06-21T01:13:41.774 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:41 smithi120 podman[92158]: 2024-06-21 01:13:41.435367897 +0000 UTC m=+0.629229666 container remove 50b38024b6884fd9fab438f7f6a209ebaa82205c74f64fd22f327fdabb6f853d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, architecture=x86_64, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, version=8, io.buildah.version=1.19.8) 2024-06-21T01:13:42.111 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:41 smithi120 podman[92250]: 2024-06-21 01:13:41.679541219 +0000 UTC m=+0.017410631 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:42.111 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:41 smithi120 podman[92250]: 2024-06-21 01:13:41.811237982 +0000 UTC m=+0.149107389 container create 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:13:42.111 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 podman[92250]: 2024-06-21 01:13:42.022554098 +0000 UTC m=+0.360423514 container init 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-06-21T01:13:42.112 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 podman[92250]: 2024-06-21 01:13:42.026150576 +0000 UTC m=+0.364019984 container start 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-21T01:13:42.112 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 podman[92250]: 2024-06-21 01:13:42.11388025 +0000 UTC m=+0.451749667 container attach 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-06-21T01:13:42.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 podman[92250]: 2024-06-21 01:13:42.288271186 +0000 UTC m=+0.626140585 container died 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-06-21T01:13:42.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:42 smithi027 ceph-mon[84934]: osdmap e155: 8 total, 7 up, 8 in 2024-06-21T01:13:42.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:42 smithi027 ceph-mon[90235]: osdmap e155: 8 total, 7 up, 8 in 2024-06-21T01:13:42.864 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:42 smithi120 ceph-mon[67716]: osdmap e155: 8 total, 7 up, 8 in 2024-06-21T01:13:43.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 podman[92274]: 2024-06-21 01:13:42.867819504 +0000 UTC m=+0.574062935 container remove 0090c21296878163f49609236e9f3f231520f9575b28b88eaa5eec269ca6046d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-06-21T01:13:43.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.7.service: Deactivated successfully. 2024-06-21T01:13:43.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 systemd[1]: Stopped Ceph osd.7 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:13:43.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:42 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.7.service: Consumed 17.262s CPU time. 2024-06-21T01:13:43.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 systemd[1]: Starting Ceph osd.7 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:13:43.610 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-mon[67716]: pgmap v229: 161 pgs: 8 active+undersized, 13 stale+active+clean, 24 peering, 5 active+undersized+degraded, 111 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-06-21T01:13:43.610 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-mon[67716]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:13:43.610 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-mon[67716]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 5 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:43.610 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 podman[92374]: 2024-06-21 01:13:43.202495495 +0000 UTC m=+0.017757073 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:43.611 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 podman[92374]: 2024-06-21 01:13:43.352322611 +0000 UTC m=+0.167584190 container create 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-06-21T01:13:43.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[84934]: pgmap v229: 161 pgs: 8 active+undersized, 13 stale+active+clean, 24 peering, 5 active+undersized+degraded, 111 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-06-21T01:13:43.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[84934]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:13:43.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[84934]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 5 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:43.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[90235]: pgmap v229: 161 pgs: 8 active+undersized, 13 stale+active+clean, 24 peering, 5 active+undersized+degraded, 111 active+clean; 585 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-06-21T01:13:43.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[90235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-21T01:13:43.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:43 smithi027 ceph-mon[90235]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 5 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:43.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 podman[92374]: 2024-06-21 01:13:43.613043774 +0000 UTC m=+0.428305363 container init 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-06-21T01:13:43.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 podman[92374]: 2024-06-21 01:13:43.616391551 +0000 UTC m=+0.431653139 container start 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-06-21T01:13:43.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 podman[92374]: 2024-06-21 01:13:43.69650302 +0000 UTC m=+0.511764602 container attach 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81) 2024-06-21T01:13:44.347 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 bash[92374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 bash[92374]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:43 smithi120 bash[92374]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 bash[92374]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 bash[92374]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-21T01:13:44.348 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-21T01:13:44.349 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 bash[92374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-21T01:13:44.349 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate[92388]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-21T01:13:44.349 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 bash[92374]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-21T01:13:44.349 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 podman[92374]: 2024-06-21 01:13:44.029850084 +0000 UTC m=+0.845111677 container died 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-06-21T01:13:45.002 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 podman[92516]: 2024-06-21 01:13:44.661152586 +0000 UTC m=+0.625601956 container remove 838ef86e4be51d25d3565e0c13d2dfa8727ae86325d4a09ea0c87cd064931261 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:13:45.002 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 podman[92574]: 2024-06-21 01:13:44.782689148 +0000 UTC m=+0.018094776 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:13:45.002 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:44 smithi120 podman[92574]: 2024-06-21 01:13:44.903753296 +0000 UTC m=+0.139158913 container create 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240618) 2024-06-21T01:13:45.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 podman[92574]: 2024-06-21 01:13:45.11460255 +0000 UTC m=+0.350008182 container init 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-06-21T01:13:45.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 podman[92574]: 2024-06-21 01:13:45.117865792 +0000 UTC m=+0.353271409 container start 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True) 2024-06-21T01:13:45.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-osd[92592]: -- 172.21.15.120:0/2962493520 <== mon.2 v2:172.21.15.120:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561c7d20fa40 con 0x561c7dfce000 2024-06-21T01:13:45.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 bash[92574]: 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 2024-06-21T01:13:45.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 systemd[1]: Started Ceph osd.7 for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[84934]: pgmap v230: 161 pgs: 11 active+undersized, 10 stale+active+clean, 24 peering, 11 active+undersized+degraded, 105 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[90235]: pgmap v230: 161 pgs: 11 active+undersized, 10 stale+active+clean, 24 peering, 11 active+undersized+degraded, 105 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:45.792 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-mon[67716]: pgmap v230: 161 pgs: 11 active+undersized, 10 stale+active+clean, 24 peering, 11 active+undersized+degraded, 105 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 30/723 objects degraded (4.149%) 2024-06-21T01:13:45.792 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.792 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:45.792 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:45.792 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:45 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:13:45.655+0000 7f3d5da0e640 -1 Falling back to public interface 2024-06-21T01:13:47.723 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:47 smithi120 ceph-mon[67716]: pgmap v231: 161 pgs: 32 active+undersized, 13 peering, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 73/723 objects degraded (10.097%) 2024-06-21T01:13:47.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:47 smithi027 ceph-mon[90235]: pgmap v231: 161 pgs: 32 active+undersized, 13 peering, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 73/723 objects degraded (10.097%) 2024-06-21T01:13:47.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:47 smithi027 ceph-mon[84934]: pgmap v231: 161 pgs: 32 active+undersized, 13 peering, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 73/723 objects degraded (10.097%) 2024-06-21T01:13:48.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 73/723 objects degraded (10.097%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:48.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-mon[67716]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-mon[67716]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:13:48.375+0000 7f3d5da0e640 -1 osd.7 0 read_superblock omap replica is missing. 2024-06-21T01:13:48.617 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:13:48.407+0000 7f3d5da0e640 -1 osd.7 153 log_to_monitors true 2024-06-21T01:13:48.617 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:13:48 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:13:48.497+0000 7f3d557b9640 -1 osd.7 153 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-21T01:13:48.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 73/723 objects degraded (10.097%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:48.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[84934]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[84934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 73/723 objects degraded (10.097%), 22 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:48.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[90235]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-mon[90235]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-21T01:13:48.729 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:48] "GET /metrics HTTP/1.1" 200 38196 "" "Prometheus/2.43.0" 2024-06-21T01:13:49.809 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: pgmap v232: 161 pgs: 39 active+undersized, 28 active+undersized+degraded, 94 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-06-21T01:13:49.809 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:13:49.809 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T01:13:49.809 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.809 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: osdmap e156: 8 total, 7 up, 8 in 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.810 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: pgmap v232: 161 pgs: 39 active+undersized, 28 active+undersized+degraded, 94 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-06-21T01:13:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: osdmap e156: 8 total, 7 up, 8 in 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: pgmap v232: 161 pgs: 39 active+undersized, 28 active+undersized+degraded, 94 active+clean; 585 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 96/723 objects degraded (13.278%) 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-21T01:13:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: osdmap e156: 8 total, 7 up, 8 in 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi120", "root=default"]}]: dispatch 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:49.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229] boot 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: osdmap e157: 8 total, 8 up, 8 in 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.703 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:50 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229] boot 2024-06-21T01:13:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: osdmap e157: 8 total, 8 up, 8 in 2024-06-21T01:13:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:13:50.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: osd.7 [v2:172.21.15.120:6824/2510596229,v1:172.21.15.120:6825/2510596229] boot 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: osdmap e157: 8 total, 8 up, 8 in 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:50.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:50 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:51.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:51 smithi120 ceph-mon[67716]: pgmap v235: 161 pgs: 11 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-06-21T01:13:51.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:51 smithi120 ceph-mon[67716]: osdmap e158: 8 total, 8 up, 8 in 2024-06-21T01:13:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:51 smithi027 ceph-mon[84934]: pgmap v235: 161 pgs: 11 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-06-21T01:13:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:51 smithi027 ceph-mon[84934]: osdmap e158: 8 total, 8 up, 8 in 2024-06-21T01:13:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:51 smithi027 ceph-mon[90235]: pgmap v235: 161 pgs: 11 peering, 34 active+undersized, 22 active+undersized+degraded, 94 active+clean; 585 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 78/723 objects degraded (10.788%) 2024-06-21T01:13:51.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:51 smithi027 ceph-mon[90235]: osdmap e158: 8 total, 8 up, 8 in 2024-06-21T01:13:53.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: pgmap v237: 161 pgs: 11 peering, 16 active+undersized, 13 active+undersized+degraded, 121 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 50/723 objects degraded (6.916%) 2024-06-21T01:13:53.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: Health check update: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:53.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-21T01:13:53.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-21T01:13:53.870 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-21T01:13:53.871 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-21T01:13:53.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-21T01:13:53.872 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:53 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-21T01:13:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: pgmap v237: 161 pgs: 11 peering, 16 active+undersized, 13 active+undersized+degraded, 121 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 50/723 objects degraded (6.916%) 2024-06-21T01:13:53.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: Health check update: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-21T01:13:53.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-21T01:13:53.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-21T01:13:53.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-21T01:13:53.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-21T01:13:53.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: pgmap v237: 161 pgs: 11 peering, 16 active+undersized, 13 active+undersized+degraded, 121 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 50/723 objects degraded (6.916%) 2024-06-21T01:13:53.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: Health check update: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded (PG_DEGRADED) 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:53.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-21T01:13:53.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-21T01:13:53.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-21T01:13:53.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-21T01:13:53.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-21T01:13:53.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-21T01:13:53.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-21T01:13:53.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-21T01:13:53.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-21T01:13:53.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-21T01:13:53.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-21T01:13:53.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-21T01:13:53.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-21T01:13:53.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-21T01:13:53.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-21T01:13:53.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-21T01:13:53.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:53 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-21T01:13:54.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all osd 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: Upgrade: Setting require_osd_release to 19 squid 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: osdmap e159: 8 total, 8 up, 8 in 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:13:54.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:54 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all osd 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: Upgrade: Setting require_osd_release to 19 squid 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: osdmap e159: 8 total, 8 up, 8 in 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:13:54.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all osd 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: Upgrade: Setting require_osd_release to 19 squid 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: osdmap e159: 8 total, 8 up, 8 in 2024-06-21T01:13:54.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:13:54.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:13:54.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:54 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:13:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mds 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: pgmap v239: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nfs 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:13:55.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.868 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded) 2024-06-21T01:13:55.869 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:55 smithi120 ceph-mon[67716]: Cluster is now healthy 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mds 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: pgmap v239: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nfs 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Finalizing container_image settings 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:13:55.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Upgrade: Complete! 2024-06-21T01:13:55.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:13:55.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:13:55.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mds 2024-06-21T01:13:55.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: pgmap v239: 161 pgs: 11 peering, 150 active+clean; 585 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-06-21T01:13:55.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nfs 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Finalizing container_image settings 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Upgrade: Complete! 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:13:55.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded) 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[84934]: Cluster is now healthy 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 50/723 objects degraded (6.916%), 13 pgs degraded) 2024-06-21T01:13:55.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:55 smithi027 ceph-mon[90235]: Cluster is now healthy 2024-06-21T01:13:57.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:57 smithi027 ceph-mon[84934]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-06-21T01:13:57.780 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:57 smithi027 ceph-mon[90235]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-06-21T01:13:57.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:57 smithi120 ceph-mon[67716]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 740 B/s rd, 0 op/s 2024-06-21T01:13:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:13:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:13:58] "GET /metrics HTTP/1.1" 200 38131 "" "Prometheus/2.43.0" 2024-06-21T01:13:59.588 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[84934]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-21T01:13:59.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:59.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:59.589 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[90235]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-21T01:13:59.589 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:59.589 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:13:59 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:13:59.827 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:59 smithi120 ceph-mon[67716]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 242 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-21T01:13:59.828 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:13:59.828 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:13:59 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:00.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:14:00.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:00 smithi027 ceph-mon[84934]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:00.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:00 smithi027 ceph-mon[90235]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:00.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:00 smithi120 ceph-mon[67716]: from='client.64507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:01.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:01 smithi027 ceph-mon[84934]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 44 op/s 2024-06-21T01:14:01.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:01 smithi027 ceph-mon[90235]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 44 op/s 2024-06-21T01:14:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:01 smithi120 ceph-mon[67716]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 44 op/s 2024-06-21T01:14:02.229 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 2m ago 30m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (8m) 12s ago 30m 80.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (9m) 2m ago 29m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (9m) 12s ago 35m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (19m) 2m ago 37m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:14:02.230 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 2m ago 37m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (7m) 12s ago 35m 50.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 2m ago 35m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 31m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (9m) 12s ago 31m 16.8M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (4m) 2m ago 35m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:14:02.231 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (4m) 2m ago 34m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (3m) 2m ago 34m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (2m) 2m ago 33m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (117s) 12s ago 33m 47.6M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (84s) 12s ago 33m 47.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (50s) 12s ago 32m 51.8M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (17s) 12s ago 32m 14.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (8m) 12s ago 31m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:14:02.232 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (30m) 2m ago 30m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:14:02.233 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (30m) 12s ago 30m 80.2M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:14:03.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-06-21T01:14:03.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:03 smithi027 ceph-mon[84934]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:03.710 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:03 smithi027 ceph-mon[84934]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 101 op/s 2024-06-21T01:14:03.710 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:03 smithi027 ceph-mon[90235]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:03.710 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:03 smithi027 ceph-mon[90235]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 101 op/s 2024-06-21T01:14:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:03 smithi120 ceph-mon[67716]: from='client.54585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:03 smithi120 ceph-mon[67716]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 0 B/s wr, 101 op/s 2024-06-21T01:14:05.286 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:14:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:05 smithi120 ceph-mon[67716]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 131 op/s 2024-06-21T01:14:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:05 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:05 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/685620102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:05.673 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.673 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[90235]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 131 op/s 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/685620102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[84934]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 246 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 131 op/s 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:05.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:05 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/685620102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:06.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-06-21T01:14:07.629 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:07 smithi027 ceph-mon[84934]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T01:14:07.629 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:07 smithi027 ceph-mon[90235]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T01:14:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:07 smithi120 ceph-mon[67716]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-06-21T01:14:08.199 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)" 2024-06-21T01:14:08.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:08 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/868652393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:08.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:08 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/868652393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:08.829 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:08] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-06-21T01:14:08.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:14:08.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:08 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/868652393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:09 smithi120 ceph-mon[67716]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T01:14:09.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:09.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:09 smithi027 ceph-mon[84934]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T01:14:09.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:09.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:09 smithi027 ceph-mon[90235]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-06-21T01:14:09.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:14:10.817 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:14:10.818 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:14:10.818 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:14:11.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:14:11.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:11 smithi027 ceph-mon[84934]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 114 op/s 2024-06-21T01:14:11.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:11 smithi027 ceph-mon[84934]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:11.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:11 smithi027 ceph-mon[90235]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 114 op/s 2024-06-21T01:14:11.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:11 smithi027 ceph-mon[90235]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:11 smithi120 ceph-mon[67716]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 114 op/s 2024-06-21T01:14:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:11 smithi120 ceph-mon[67716]: from='client.54603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:13.651 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:14:13.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:13 smithi027 ceph-mon[84934]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 84 op/s 2024-06-21T01:14:13.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:13 smithi027 ceph-mon[90235]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 84 op/s 2024-06-21T01:14:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:13 smithi120 ceph-mon[67716]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 84 op/s 2024-06-21T01:14:14.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-06-21T01:14:14.705 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:14 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/898685136' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:14:14.705 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:14 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/898685136' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:14:14.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:14 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/898685136' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:14:15.677 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:15 smithi027 ceph-mon[84934]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 35 op/s 2024-06-21T01:14:15.678 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:15 smithi027 ceph-mon[90235]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 35 op/s 2024-06-21T01:14:15.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:15 smithi120 ceph-mon[67716]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 35 op/s 2024-06-21T01:14:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:17 smithi027 ceph-mon[84934]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:14:17.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:17 smithi027 ceph-mon[84934]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:17.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:17 smithi027 ceph-mon[90235]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:14:17.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:17 smithi027 ceph-mon[90235]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:17 smithi120 ceph-mon[67716]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-21T01:14:17.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:17 smithi120 ceph-mon[67716]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:18.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:18] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-06-21T01:14:19.118 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:19.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-06-21T01:14:20.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:20.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:20.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:20.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:20.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.408 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:20.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:20.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:20.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:20.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:20.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:20.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:20.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:20.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:21.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:21 smithi027 ceph-mon[84934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:21.463 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:21 smithi027 ceph-mon[90235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:21.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:21 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:14:22.311 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:14:22.311 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:22 smithi027 ceph-mon[84934]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:22.311 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:22 smithi027 ceph-mon[90235]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:22.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:22 smithi120 ceph-mon[67716]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:23.009 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:14:23.009 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 2m ago 31m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:14:23.009 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (8m) 33s ago 30m 80.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:14:23.009 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (9m) 2m ago 30m 60.3M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (10m) 33s ago 35m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (20m) 2m ago 37m 504M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 2m ago 37m 52.9M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (7m) 33s ago 36m 50.0M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 2m ago 36m 39.5M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 31m 16.2M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (9m) 33s ago 31m 16.8M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (5m) 2m ago 35m 48.5M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (4m) 2m ago 35m 45.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (3m) 2m ago 34m 42.6M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (2m) 2m ago 34m 12.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (2m) 33s ago 33m 47.6M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (104s) 33s ago 33m 47.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:14:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (71s) 33s ago 33m 51.8M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:14:23.011 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (38s) 33s ago 32m 14.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:14:23.011 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (9m) 33s ago 32m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:14:23.011 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (30m) 2m ago 30m 81.5M - 17.2.0 e1d6a67b021e 5d241db83c3b 2024-06-21T01:14:23.011 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (30m) 33s ago 30m 80.2M - 17.2.0 e1d6a67b021e 8d9badfa5c19 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:14:23.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 8 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 13 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:14:23.445 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:14:23.802 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:14:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/2 daemons upgraded", 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading rgw daemons", 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:14:23.803 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:14:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mgr 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all osd 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mds 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='client.54633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2128919404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mgr 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all osd 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mds 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='client.54633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:23 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2128919404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='client.64549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='client.64555 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all osd 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mds 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='client.54633 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.368 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:23 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2128919404' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[84934]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:24.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:24.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[90235]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:24.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:24.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:24.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:24 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:24 smithi120 ceph-mon[67716]: from='client.54645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:24 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:24 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.dsjkpz", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:24 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:25.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[84934]: Upgrade: Updating rgw.foo.smithi027.dsjkpz (1/2) 2024-06-21T01:14:25.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[84934]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:14:25.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[84934]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[90235]: Upgrade: Updating rgw.foo.smithi027.dsjkpz (1/2) 2024-06-21T01:14:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[90235]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:14:25.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:25 smithi027 ceph-mon[90235]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:25 smithi120 ceph-mon[67716]: Upgrade: Updating rgw.foo.smithi027.dsjkpz (1/2) 2024-06-21T01:14:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:25 smithi120 ceph-mon[67716]: Deploying daemon rgw.foo.smithi027.dsjkpz on smithi027 2024-06-21T01:14:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:25 smithi120 ceph-mon[67716]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:28.085 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:27 smithi027 ceph-mon[84934]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:28.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:27 smithi027 ceph-mon[90235]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:28.204 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:27 smithi120 ceph-mon[67716]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:14:28.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:28] "GET /metrics HTTP/1.1" 200 38285 "" "Prometheus/2.43.0" 2024-06-21T01:14:29.450 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:29 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:29.450 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:29 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:29.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:29 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:29.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:29 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:29.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:29 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:29.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:29 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:30.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:30 smithi120 ceph-mon[67716]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:30.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:30 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:30.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:30.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:30.176 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:30.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[84934]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:30.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:30.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[90235]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 250 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi120.bfbgme", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-21T01:14:30.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:31.198 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:31 smithi120 ceph-mon[67716]: Upgrade: Updating rgw.foo.smithi120.bfbgme (2/2) 2024-06-21T01:14:31.198 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:31 smithi120 ceph-mon[67716]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:14:31.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:31 smithi027 ceph-mon[84934]: Upgrade: Updating rgw.foo.smithi120.bfbgme (2/2) 2024-06-21T01:14:31.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:31 smithi027 ceph-mon[84934]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:14:31.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:31 smithi027 ceph-mon[90235]: Upgrade: Updating rgw.foo.smithi120.bfbgme (2/2) 2024-06-21T01:14:31.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:31 smithi027 ceph-mon[90235]: Deploying daemon rgw.foo.smithi120.bfbgme on smithi120 2024-06-21T01:14:32.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:32 smithi120 ceph-mon[67716]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 60 op/s 2024-06-21T01:14:32.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:32 smithi027 ceph-mon[84934]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 60 op/s 2024-06-21T01:14:32.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:32 smithi027 ceph-mon[90235]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 85 B/s wr, 60 op/s 2024-06-21T01:14:33.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:33 smithi027 ceph-mon[90235]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 85 B/s wr, 106 op/s 2024-06-21T01:14:33.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:33 smithi027 ceph-mon[84934]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 85 B/s wr, 106 op/s 2024-06-21T01:14:33.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:33 smithi120 ceph-mon[67716]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 85 B/s wr, 106 op/s 2024-06-21T01:14:34.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:34.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:34.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:34.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:34.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:35 smithi027 ceph-mon[84934]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 85 B/s wr, 130 op/s 2024-06-21T01:14:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:35 smithi027 ceph-mon[90235]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 85 B/s wr, 130 op/s 2024-06-21T01:14:35.760 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:35 smithi120 ceph-mon[67716]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 85 B/s wr, 130 op/s 2024-06-21T01:14:38.111 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 170 B/s wr, 177 op/s 2024-06-21T01:14:38.111 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.111 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.111 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.111 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.112 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.112 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:37 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 170 B/s wr, 177 op/s 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 258 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s rd, 170 B/s wr, 177 op/s 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:37 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:38.904 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:38] "GET /metrics HTTP/1.1" 200 38281 "" "Prometheus/2.43.0" 2024-06-21T01:14:39.166 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:38 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:39.166 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:38 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:38 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:39.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:38 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:39.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:38 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:39.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:38 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:39 smithi027 ceph-mon[84934]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 235 op/s 2024-06-21T01:14:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:39 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:40.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:39 smithi027 ceph-mon[90235]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 235 op/s 2024-06-21T01:14:40.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:39 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:40.273 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:39 smithi120 ceph-mon[67716]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 262 MiB used, 715 GiB / 715 GiB avail; 149 KiB/s rd, 170 B/s wr, 235 op/s 2024-06-21T01:14:40.274 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:39 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-21T01:14:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]': finished 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]': finished 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:14:41.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-21T01:14:41.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]: dispatch 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]': finished 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]: dispatch 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]': finished 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:14:41.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:41.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:41 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 247 op/s 2024-06-21T01:14:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.dsjkpz"}]': finished 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi120.bfbgme"}]': finished 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-21T01:14:42.119 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-21T01:14:42.120 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-21T01:14:42.121 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-21T01:14:42.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-21T01:14:42.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:42.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:42.122 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:41 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rgw 2024-06-21T01:14:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:14:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nfs 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Finalizing container_image settings 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Upgrade: Complete! 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: Checking dashboard <-> RGW credentials 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rgw 2024-06-21T01:14:42.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nfs 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Finalizing container_image settings 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Upgrade: Complete! 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: Checking dashboard <-> RGW credentials 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:42.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:42 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rgw 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nfs 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all nvmeof 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Finalizing container_image settings 2024-06-21T01:14:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Upgrade: Complete! 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: Checking dashboard <-> RGW credentials 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:14:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:42 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:43 smithi120 ceph-mon[67716]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 85 B/s wr, 188 op/s 2024-06-21T01:14:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:43 smithi027 ceph-mon[84934]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 85 B/s wr, 188 op/s 2024-06-21T01:14:44.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:43 smithi027 ceph-mon[90235]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 85 B/s wr, 188 op/s 2024-06-21T01:14:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:45 smithi027 ceph-mon[84934]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 85 B/s wr, 142 op/s 2024-06-21T01:14:45.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:45 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:45 smithi027 ceph-mon[90235]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 85 B/s wr, 142 op/s 2024-06-21T01:14:45.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:45 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:45 smithi120 ceph-mon[67716]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 85 B/s wr, 142 op/s 2024-06-21T01:14:45.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:45 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:14:47.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:47 smithi027 ceph-mon[84934]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-06-21T01:14:47.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:47 smithi027 ceph-mon[90235]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-06-21T01:14:47.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:47 smithi120 ceph-mon[67716]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 85 B/s wr, 119 op/s 2024-06-21T01:14:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:48] "GET /metrics HTTP/1.1" 200 38281 "" "Prometheus/2.43.0" 2024-06-21T01:14:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:49 smithi120 ceph-mon[67716]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-06-21T01:14:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:49 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:49.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[84934]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-06-21T01:14:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:49.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[90235]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 0 B/s wr, 73 op/s 2024-06-21T01:14:49.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:14:51.833 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:51 smithi027 ceph-mon[84934]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 15 op/s 2024-06-21T01:14:51.834 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:51 smithi027 ceph-mon[90235]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 15 op/s 2024-06-21T01:14:51.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:51 smithi120 ceph-mon[67716]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 0 B/s wr, 15 op/s 2024-06-21T01:14:53.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:53 smithi120 ceph-mon[67716]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-21T01:14:53.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:53 smithi027 ceph-mon[84934]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-21T01:14:53.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:53 smithi027 ceph-mon[90235]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-06-21T01:14:54.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:14:55.514 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:55 smithi027 ceph-mon[84934]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:55.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:55 smithi027 ceph-mon[84934]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-06-21T01:14:55.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:55 smithi027 ceph-mon[90235]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:55.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:55 smithi027 ceph-mon[90235]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-06-21T01:14:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:55 smithi120 ceph-mon[67716]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:55.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:55 smithi120 ceph-mon[67716]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 19s ago 31m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (9m) 20s ago 31m 81.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (10m) 19s ago 30m 60.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (10m) 20s ago 36m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (20m) 19s ago 37m 510M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (7m) 19s ago 38m 61.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (8m) 20s ago 36m 51.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 19s ago 36m 44.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:14:57.091 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (10m) 19s ago 32m 16.3M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (10m) 20s ago 32m 16.3M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (5m) 19s ago 36m 57.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (5m) 19s ago 35m 56.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (4m) 19s ago 35m 50.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (3m) 19s ago 34m 62.4M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (2m) 20s ago 34m 54.2M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (2m) 20s ago 34m 52.0M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (105s) 20s ago 33m 57.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (72s) 20s ago 33m 51.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (9m) 20s ago 32m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (29s) 19s ago 31m 76.7M - 19.0.0-2541-gc65be810 e9bfc9e474e5 38443b4bd372 2024-06-21T01:14:57.092 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (24s) 20s ago 30m 76.5M - 19.0.0-2541-gc65be810 e9bfc9e474e5 2fa9173a2ba1 2024-06-21T01:14:57.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:57 smithi027 ceph-mon[84934]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-06-21T01:14:57.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:57 smithi027 ceph-mon[84934]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:57.860 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:57 smithi027 ceph-mon[90235]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-06-21T01:14:57.860 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:57 smithi027 ceph-mon[90235]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:57.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:57 smithi120 ceph-mon[67716]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-06-21T01:14:57.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:57 smithi120 ceph-mon[67716]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:14:57.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-06-21T01:14:58.821 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:14:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:14:58] "GET /metrics HTTP/1.1" 200 38315 "" "Prometheus/2.43.0" 2024-06-21T01:14:59.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:59 smithi027 ceph-mon[84934]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:14:59.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:14:59 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:59.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:59 smithi027 ceph-mon[90235]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:14:59.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:14:59 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:59 smithi120 ceph-mon[67716]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-06-21T01:14:59.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:14:59 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:14:59.993 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:15:00.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:00 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/711680545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:00.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:00 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/711680545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:00.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-06-21T01:15:00.821 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:00 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/711680545' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:01.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:01 smithi027 ceph-mon[84934]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:01.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:01 smithi027 ceph-mon[90235]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:01.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:01 smithi120 ceph-mon[67716]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:02.979 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)" 2024-06-21T01:15:03.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:03 smithi027 ceph-mon[84934]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:03.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:03 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3593638601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:03.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:03 smithi027 ceph-mon[90235]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:03.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:03 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3593638601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:03.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:15:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:03 smithi120 ceph-mon[67716]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:03.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:03 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3593638601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:04.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:04.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:04.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:05.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:05 smithi027 ceph-mon[84934]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:05.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:05 smithi027 ceph-mon[90235]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:05.820 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:15:05.821 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:15:05.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:05 smithi120 ceph-mon[67716]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:06.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:15:06.855 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:06 smithi027 ceph-mon[84934]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:06.855 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:06 smithi027 ceph-mon[90235]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:06.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:06 smithi120 ceph-mon[67716]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:07.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:07 smithi120 ceph-mon[67716]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:07 smithi027 ceph-mon[84934]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:07.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:07 smithi027 ceph-mon[90235]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:08.779 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:15:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:08] "GET /metrics HTTP/1.1" 200 38313 "" "Prometheus/2.43.0" 2024-06-21T01:15:09.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[84934]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1445329328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[90235]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1445329328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:09.852 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:09 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:09.865 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:09 smithi120 ceph-mon[67716]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:09 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1445329328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:09.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:09 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:11.605 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:11.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:11 smithi027 ceph-mon[84934]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:11.606 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:11 smithi027 ceph-mon[90235]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:11.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:11 smithi120 ceph-mon[67716]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:12.362 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-21T01:15:12.372 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-06-21T01:15:12.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-06-21T01:15:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:12.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.867 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:12 smithi120 ceph-mon[67716]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.908 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[90235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='client.54759 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:12 smithi027 ceph-mon[84934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd 2024-06-21T01:15:13.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:13 smithi120 ceph-mon[67716]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:13.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:13 smithi027 ceph-mon[84934]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:13.891 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:13 smithi027 ceph-mon[90235]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:14.590 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 37s ago 32m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (9m) 38s ago 31m 81.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (10m) 37s ago 31m 60.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (10m) 38s ago 36m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (21m) 37s ago 38m 510M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (7m) 37s ago 38m 61.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (8m) 38s ago 36m 51.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 37s ago 37m 44.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:15:15.280 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (10m) 37s ago 32m 16.3M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (10m) 38s ago 32m 16.3M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (6m) 37s ago 36m 57.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (5m) 37s ago 35m 56.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (4m) 37s ago 35m 50.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (3m) 37s ago 35m 62.4M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (3m) 38s ago 34m 54.2M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (2m) 38s ago 34m 52.0M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (2m) 38s ago 33m 57.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (90s) 38s ago 33m 51.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (10m) 38s ago 32m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (47s) 37s ago 31m 76.7M - 19.0.0-2541-gc65be810 e9bfc9e474e5 38443b4bd372 2024-06-21T01:15:15.281 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (42s) 38s ago 31m 76.5M - 19.0.0-2541-gc65be810 e9bfc9e474e5 2fa9173a2ba1 2024-06-21T01:15:15.710 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 8 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:15.711 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:15:15.712 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 15 2024-06-21T01:15:15.712 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:15:15.712 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:15:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:15.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:15.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:15 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.068 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "osd", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "mon", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw" 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "15/21 daemons upgraded", 2024-06-21T01:15:16.069 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading iscsi daemons", 2024-06-21T01:15:16.070 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-06-21T01:15:16.070 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='client.64705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:15 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.505 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-06-21T01:15:16.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='client.54777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:15:16.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mgr 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mon 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all crash 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all osd 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all mds 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rgw 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2016318841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='client.64723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3103287204' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='client.54777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:15:16.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mgr 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mon 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all crash 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all osd 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all mds 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rgw 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2016318841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='client.64723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:16.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:16 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3103287204' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='client.54777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Target is version 19.0.0-2541-gc65be810 (squid) 2024-06-21T01:15:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, digests ['quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0'] 2024-06-21T01:15:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mgr 2024-06-21T01:15:17.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mon 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all crash 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all osd 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all mds 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rgw 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all rbd-mirror 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all cephfs-mirror 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: Upgrade: Setting container_image for all ceph-exporter 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2016318841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='client.64723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:17.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:16 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3103287204' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: Upgrade: Updating iscsi.foo.smithi027.temphd 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:18.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:18 smithi120 ceph-mon[67716]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:15:18.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:18.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: Upgrade: Updating iscsi.foo.smithi027.temphd 2024-06-21T01:15:18.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:18.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[84934]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: Upgrade: Updating iscsi.foo.smithi027.temphd 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi027.temphd", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-mon[90235]: Deploying daemon iscsi.foo.smithi027.temphd on smithi027 2024-06-21T01:15:18.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:18] "GET /metrics HTTP/1.1" 200 38313 "" "Prometheus/2.43.0" 2024-06-21T01:15:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:20 smithi120 ceph-mon[67716]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:20.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:20 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:20.367 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:20 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[90235]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[84934]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:20.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:20 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:22 smithi120 ceph-mon[67716]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:22.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:22 smithi027 ceph-mon[84934]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:22.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:22 smithi027 ceph-mon[90235]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:24 smithi120 ceph-mon[67716]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:24.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:24 smithi027 ceph-mon[84934]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:24.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:24 smithi027 ceph-mon[90235]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:26 smithi120 ceph-mon[67716]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:26.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:26 smithi027 ceph-mon[84934]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:26.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:26 smithi027 ceph-mon[90235]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:28 smithi120 ceph-mon[67716]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:28.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:28 smithi027 ceph-mon[84934]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:28.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:28 smithi027 ceph-mon[90235]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:28.851 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:28] "GET /metrics HTTP/1.1" 200 38317 "" "Prometheus/2.43.0" 2024-06-21T01:15:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:30 smithi027 ceph-mon[84934]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:30.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:30 smithi027 ceph-mon[84934]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:30 smithi027 ceph-mon[90235]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:30.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:30 smithi027 ceph-mon[90235]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:30 smithi120 ceph-mon[67716]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:30.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:30 smithi120 ceph-mon[67716]: from='client.15585 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:30.929 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:30 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:15:30.554+0000 7f3f39d2b640 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s 2024-06-21T01:15:31.609 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:31.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:31.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[84934]: Health check failed: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-21T01:15:31.610 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:31.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:31 smithi027 ceph-mon[90235]: Health check failed: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s 2024-06-21T01:15:31.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:15:31.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:15:31.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:31.617 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:31 smithi120 ceph-mon[67716]: Health check failed: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-06-21T01:15:33.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:33 smithi120 ceph-mon[67716]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:33.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:33 smithi027 ceph-mon[90235]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:33.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:33 smithi027 ceph-mon[84934]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:35 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:35.616 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:35 smithi120 ceph-mon[67716]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[90235]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:35.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:35 smithi027 ceph-mon[84934]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 807 B/s rd, 0 op/s 2024-06-21T01:15:37.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:37 smithi120 ceph-mon[67716]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:37.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:37 smithi027 ceph-mon[84934]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:37.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:37 smithi027 ceph-mon[90235]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:38.965 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:38] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-06-21T01:15:39.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:39 smithi027 ceph-mon[84934]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:39.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:39 smithi027 ceph-mon[90235]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:39 smithi120 ceph-mon[67716]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:41.808 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:41 smithi027 ceph-mon[84934]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:41.808 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:41 smithi027 ceph-mon[90235]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:41.866 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:41 smithi120 ceph-mon[67716]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-06-21T01:15:42.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:42 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/263412281' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:15:42.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:42 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]: dispatch 2024-06-21T01:15:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:42 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/263412281' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:15:42.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:42 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]: dispatch 2024-06-21T01:15:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:42 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/263412281' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-06-21T01:15:43.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:42 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]: dispatch 2024-06-21T01:15:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[90235]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T01:15:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]': finished 2024-06-21T01:15:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[90235]: osdmap e160: 8 total, 8 up, 8 in 2024-06-21T01:15:43.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]: dispatch 2024-06-21T01:15:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[84934]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T01:15:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]': finished 2024-06-21T01:15:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[84934]: osdmap e160: 8 total, 8 up, 8 in 2024-06-21T01:15:43.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:43 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]: dispatch 2024-06-21T01:15:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:43 smithi120 ceph-mon[67716]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-21T01:15:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:43 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3415049219' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/1151429623"}]': finished 2024-06-21T01:15:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:43 smithi120 ceph-mon[67716]: osdmap e160: 8 total, 8 up, 8 in 2024-06-21T01:15:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:43 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]: dispatch 2024-06-21T01:15:44.616 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:15:44 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=cleanup t=2024-06-21T01:15:44.223431355Z level=info msg="Completed cleanup jobs" duration=1.185676ms 2024-06-21T01:15:44.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]': finished 2024-06-21T01:15:44.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[84934]: osdmap e161: 8 total, 8 up, 8 in 2024-06-21T01:15:44.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]: dispatch 2024-06-21T01:15:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]': finished 2024-06-21T01:15:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[90235]: osdmap e161: 8 total, 8 up, 8 in 2024-06-21T01:15:44.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:44 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]: dispatch 2024-06-21T01:15:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:44 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1886576508' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/2296470170"}]': finished 2024-06-21T01:15:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:44 smithi120 ceph-mon[67716]: osdmap e161: 8 total, 8 up, 8 in 2024-06-21T01:15:45.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:44 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]: dispatch 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[90235]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]': finished 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[90235]: osdmap e162: 8 total, 8 up, 8 in 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]: dispatch 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[84934]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:15:45.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]': finished 2024-06-21T01:15:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[84934]: osdmap e162: 8 total, 8 up, 8 in 2024-06-21T01:15:45.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:45 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]: dispatch 2024-06-21T01:15:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:45 smithi120 ceph-mon[67716]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail 2024-06-21T01:15:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:45 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3875645086' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/422905243"}]': finished 2024-06-21T01:15:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:45 smithi120 ceph-mon[67716]: osdmap e162: 8 total, 8 up, 8 in 2024-06-21T01:15:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:45 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]: dispatch 2024-06-21T01:15:46.892 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:15:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]': finished 2024-06-21T01:15:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[84934]: osdmap e163: 8 total, 8 up, 8 in 2024-06-21T01:15:46.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]: dispatch 2024-06-21T01:15:46.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]': finished 2024-06-21T01:15:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[90235]: osdmap e163: 8 total, 8 up, 8 in 2024-06-21T01:15:46.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:46 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]: dispatch 2024-06-21T01:15:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:46 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2380067220' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6800/2254908843"}]': finished 2024-06-21T01:15:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:46 smithi120 ceph-mon[67716]: osdmap e163: 8 total, 8 up, 8 in 2024-06-21T01:15:47.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:46 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]: dispatch 2024-06-21T01:15:47.240 INFO:teuthology.orchestra.run.smithi027.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed." 2024-06-21T01:15:47.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]': finished 2024-06-21T01:15:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: osdmap e164: 8 total, 8 up, 8 in 2024-06-21T01:15:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: from='client.54855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/271346216' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:47.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[90235]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]': finished 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: osdmap e164: 8 total, 8 up, 8 in 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: from='client.54855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/271346216' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:47.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:47 smithi027 ceph-mon[84934]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:48.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1489943735' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:6801/2254908843"}]': finished 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: osdmap e164: 8 total, 8 up, 8 in 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: from='client.54855 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/271346216' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:47 smithi120 ceph-mon[67716]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]: dispatch 2024-06-21T01:15:48.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[90235]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:48.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[90235]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]': finished 2024-06-21T01:15:48.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[90235]: osdmap e165: 8 total, 8 up, 8 in 2024-06-21T01:15:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[84934]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[84934]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]': finished 2024-06-21T01:15:48.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-mon[84934]: osdmap e165: 8 total, 8 up, 8 in 2024-06-21T01:15:48.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:48] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-06-21T01:15:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:48 smithi120 ceph-mon[67716]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:48 smithi120 ceph-mon[67716]: from='client.? ' entity='client.iscsi.foo.smithi027.temphd' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.27:0/714737350"}]': finished 2024-06-21T01:15:49.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:48 smithi120 ceph-mon[67716]: osdmap e165: 8 total, 8 up, 8 in 2024-06-21T01:15:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:49 smithi027 ceph-mon[84934]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:49.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:49 smithi027 ceph-mon[90235]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:49.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:49 smithi120 ceph-mon[67716]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:15:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:15:50.248 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:15:50.248 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (10m) 72s ago 32m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:15:50.248 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (10m) 73s ago 32m 81.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (11m) 72s ago 31m 60.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (11m) 73s ago 37m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (21m) 72s ago 38m 510M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (8m) 72s ago 38m 61.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (9m) 73s ago 37m 51.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (8m) 72s ago 37m 44.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (11m) 72s ago 33m 16.3M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (11m) 73s ago 33m 16.3M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (6m) 72s ago 36m 57.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (6m) 72s ago 36m 56.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (5m) 72s ago 36m 50.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (4m) 72s ago 35m 62.4M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (3m) 73s ago 35m 54.2M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:15:50.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (3m) 73s ago 34m 52.0M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:15:50.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (2m) 73s ago 34m 57.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:15:50.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (2m) 73s ago 34m 51.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:15:50.250 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (10m) 73s ago 33m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:15:50.250 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (82s) 72s ago 31m 76.7M - 19.0.0-2541-gc65be810 e9bfc9e474e5 38443b4bd372 2024-06-21T01:15:50.250 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (77s) 73s ago 31m 76.5M - 19.0.0-2541-gc65be810 e9bfc9e474e5 2fa9173a2ba1 2024-06-21T01:15:50.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T01:15:51.972 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:51 smithi027 ceph-mon[84934]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:51.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:51 smithi027 ceph-mon[84934]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-06-21T01:15:51.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:51 smithi027 ceph-mon[90235]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:51.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:51 smithi027 ceph-mon[90235]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-06-21T01:15:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:51 smithi120 ceph-mon[67716]: from='client.64807 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:15:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:51 smithi120 ceph-mon[67716]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 1 op/s 2024-06-21T01:15:52.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:52 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:52.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:52 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 8 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:53.069 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 15 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:15:53.070 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:15:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:52 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:15:53.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-06-21T01:15:53.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:53 smithi027 ceph-mon[84934]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:15:53.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:53 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1893579253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:53.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:53 smithi027 ceph-mon[90235]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:15:53.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:53 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1893579253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:53 smithi120 ceph-mon[67716]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-06-21T01:15:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:53 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1893579253' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:15:55.534 INFO:teuthology.orchestra.run.smithi027.stdout:wait for servicemap items w/ changing names to refresh 2024-06-21T01:15:55.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:55 smithi027 ceph-mon[84934]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:15:55.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:55 smithi027 ceph-mon[90235]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:15:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:55 smithi120 ceph-mon[67716]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-06-21T01:15:56.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'sleep 60' 2024-06-21T01:15:57.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:57 smithi027 ceph-mon[84934]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:15:57.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:57 smithi027 ceph-mon[90235]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:15:58.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:57 smithi120 ceph-mon[67716]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-06-21T01:15:58.746 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:15:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:15:58] "GET /metrics HTTP/1.1" 200 38396 "" "Prometheus/2.43.0" 2024-06-21T01:15:59.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:15:59 smithi027 ceph-mon[84934]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-06-21T01:15:59.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:15:59 smithi027 ceph-mon[90235]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-06-21T01:16:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:15:59 smithi120 ceph-mon[67716]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 939 B/s rd, 0 op/s 2024-06-21T01:16:01.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:01 smithi027 ceph-mon[84934]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:01.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:01 smithi027 ceph-mon[90235]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:01 smithi120 ceph-mon[67716]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:02.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:02 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:02.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:02 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:02 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:03.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:03 smithi027 ceph-mon[90235]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:03 smithi027 ceph-mon[84934]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:03 smithi120 ceph-mon[67716]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:04.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:04.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:05.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:05 smithi027 ceph-mon[84934]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:05.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:05 smithi027 ceph-mon[90235]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:05 smithi120 ceph-mon[67716]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:07.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:07 smithi027 ceph-mon[84934]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:07.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:07 smithi027 ceph-mon[90235]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:07 smithi120 ceph-mon[67716]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:08.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:08] "GET /metrics HTTP/1.1" 200 38393 "" "Prometheus/2.43.0" 2024-06-21T01:16:09.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:09 smithi027 ceph-mon[84934]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:09 smithi027 ceph-mon[90235]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:10.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:09 smithi120 ceph-mon[67716]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:11.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:11 smithi027 ceph-mon[84934]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:11.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:11 smithi027 ceph-mon[90235]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:11 smithi120 ceph-mon[67716]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:12.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:12 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:12.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:12 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:12 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:13.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:13 smithi027 ceph-mon[84934]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:13.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:13 smithi027 ceph-mon[90235]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:13 smithi120 ceph-mon[67716]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:15 smithi120 ceph-mon[67716]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:16.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:15 smithi027 ceph-mon[84934]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:16.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:15 smithi027 ceph-mon[90235]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:17 smithi120 ceph-mon[67716]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:18.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:17 smithi027 ceph-mon[84934]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:18.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:17 smithi027 ceph-mon[90235]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:18.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:18] "GET /metrics HTTP/1.1" 200 38393 "" "Prometheus/2.43.0" 2024-06-21T01:16:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:19 smithi120 ceph-mon[67716]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:20.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:19 smithi027 ceph-mon[84934]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:20.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:19 smithi027 ceph-mon[90235]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:20.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:21.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:21 smithi027 ceph-mon[84934]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:21.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:21 smithi027 ceph-mon[90235]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:22.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:21 smithi120 ceph-mon[67716]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:23.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:22 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:23.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:22 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:23.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:22 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:24.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:23 smithi120 ceph-mon[67716]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:24.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:23 smithi027 ceph-mon[84934]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:24.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:23 smithi027 ceph-mon[90235]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:26.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:25 smithi120 ceph-mon[67716]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:26.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:25 smithi027 ceph-mon[84934]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:26.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:25 smithi027 ceph-mon[90235]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:28.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:27 smithi027 ceph-mon[84934]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:28.030 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:27 smithi027 ceph-mon[90235]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:28.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:27 smithi120 ceph-mon[67716]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:28] "GET /metrics HTTP/1.1" 200 38393 "" "Prometheus/2.43.0" 2024-06-21T01:16:30.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:29 smithi120 ceph-mon[67716]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:29 smithi027 ceph-mon[84934]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:29 smithi027 ceph-mon[90235]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:31.017 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:30 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:30 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:30 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-21T01:16:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[84934]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:16:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:16:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:16:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[90235]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:16:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:16:31.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:31 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:16:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:31 smithi120 ceph-mon[67716]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-21T01:16:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-21T01:16:32.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:31 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' 2024-06-21T01:16:33.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:32 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:33.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:32 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:33.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:32 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:34.117 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:33 smithi120 ceph-mon[67716]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:34.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:33 smithi027 ceph-mon[84934]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:34.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:33 smithi027 ceph-mon[90235]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:35.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:34 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:35.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:34 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:35.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:34 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:36.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:35 smithi120 ceph-mon[67716]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:36.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:35 smithi027 ceph-mon[84934]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:36.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:35 smithi027 ceph-mon[90235]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:38.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:37 smithi120 ceph-mon[67716]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:38.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:37 smithi027 ceph-mon[84934]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:38.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:37 smithi027 ceph-mon[90235]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:38.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:38 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:38] "GET /metrics HTTP/1.1" 200 38396 "" "Prometheus/2.43.0" 2024-06-21T01:16:40.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:39 smithi120 ceph-mon[67716]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:40.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:39 smithi027 ceph-mon[84934]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:40.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:39 smithi027 ceph-mon[90235]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:41 smithi027 ceph-mon[84934]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:41 smithi027 ceph-mon[90235]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:42.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:41 smithi120 ceph-mon[67716]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:43.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:42 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:43.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:42 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:43.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:42 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:44.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:43 smithi120 ceph-mon[67716]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:44.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:43 smithi027 ceph-mon[84934]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:44.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:43 smithi027 ceph-mon[90235]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:46.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:45 smithi120 ceph-mon[67716]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:46.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:45 smithi027 ceph-mon[84934]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:46.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:45 smithi027 ceph-mon[90235]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:48.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:47 smithi120 ceph-mon[67716]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:48.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:47 smithi027 ceph-mon[84934]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:48.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:47 smithi027 ceph-mon[90235]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:48.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:48 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:48] "GET /metrics HTTP/1.1" 200 38396 "" "Prometheus/2.43.0" 2024-06-21T01:16:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:49 smithi120 ceph-mon[67716]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:50.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:49 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:49 smithi027 ceph-mon[84934]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:50.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:49 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:49 smithi027 ceph-mon[90235]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:50.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:49 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:16:51.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:51 smithi027 ceph-mon[84934]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:51.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:51 smithi027 ceph-mon[90235]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:52.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:51 smithi120 ceph-mon[67716]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:53.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:52 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:53.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:52 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:53.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:52 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:16:54.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:53 smithi120 ceph-mon[67716]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:54.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:53 smithi027 ceph-mon[84934]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:54.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:53 smithi027 ceph-mon[90235]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:56.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:55 smithi120 ceph-mon[67716]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:56.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:55 smithi027 ceph-mon[84934]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:56.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:55 smithi027 ceph-mon[90235]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:16:58.099 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:57 smithi027 ceph-mon[84934]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:58.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:57 smithi027 ceph-mon[90235]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:58.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:57 smithi120 ceph-mon[67716]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:16:58.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:16:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:16:58] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-06-21T01:16:58.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ps' 2024-06-21T01:17:00.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:16:59 smithi120 ceph-mon[67716]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:00.219 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:16:59 smithi027 ceph-mon[84934]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:00.219 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:16:59 smithi027 ceph-mon[90235]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (11m) 2m ago 33m 27.3M - 0.25.0 c8568f914cd2 32c04beeb647 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi120 *:3000 running (11m) 2m ago 33m 81.2M - 9.4.12 2bacad6d85d8 8bdfdc7ff2cd 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi027.temphd smithi027 running (12m) 2m ago 32m 60.5M - 3.5 e1d6a67b021e bde1252a0fd5 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi120 *:8443,9283,8765 running (12m) 2m ago 38m 442M - 19.0.0-2541-gc65be810 e9bfc9e474e5 277873bc7526 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (22m) 2m ago 40m 510M - 19.0.0-2541-gc65be810 e9bfc9e474e5 16f1f10d1db0 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 2m ago 40m 61.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65d2d766d07e 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi120 running (10m) 2m ago 38m 51.4M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 65c76c6e776b 2024-06-21T01:17:00.936 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 2m ago 38m 44.8M 2048M 19.0.0-2541-gc65be810 e9bfc9e474e5 7b7984c9e8ba 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 2m ago 34m 16.3M - 1.5.0 0da6a335fe13 83944b7f409e 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi120 *:9100 running (12m) 2m ago 34m 16.3M - 1.5.0 0da6a335fe13 b727660fc48b 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 2m ago 38m 57.9M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 fb7aea189c7b 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 2m ago 37m 56.1M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 eb18d2365a85 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (6m) 2m ago 37m 50.2M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 c604eb1e6396 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (5m) 2m ago 36m 62.4M 2992M 19.0.0-2541-gc65be810 e9bfc9e474e5 6e5b1a5a8360 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi120 running (4m) 2m ago 36m 54.2M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 c47340426cbf 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi120 running (4m) 2m ago 36m 52.0M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 a8c0ce0059e8 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi120 running (3m) 2m ago 35m 57.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 cb59bc5cec74 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi120 running (3m) 2m ago 35m 51.4M 3248M 19.0.0-2541-gc65be810 e9bfc9e474e5 5098c861dd5c 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi120 *:9095 running (11m) 2m ago 34m 71.7M - 2.43.0 a07b618ecd1d a0fa63526514 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.dsjkpz smithi027 *:8000 running (2m) 2m ago 33m 76.7M - 19.0.0-2541-gc65be810 e9bfc9e474e5 38443b4bd372 2024-06-21T01:17:00.937 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi120.bfbgme smithi120 *:8000 running (2m) 2m ago 33m 76.5M - 19.0.0-2541-gc65be810 e9bfc9e474e5 2fa9173a2ba1 2024-06-21T01:17:01.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions' 2024-06-21T01:17:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:01 smithi120 ceph-mon[67716]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:02.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:01 smithi120 ceph-mon[67716]: from='client.54885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:02.147 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:01 smithi027 ceph-mon[84934]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:02.147 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:01 smithi027 ceph-mon[84934]: from='client.54885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:02.147 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:01 smithi027 ceph-mon[90235]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:02.147 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:01 smithi027 ceph-mon[90235]: from='client.54885 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:03.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:02 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:03.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:02 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:03.180 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:02 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:03.723 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 3 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 8 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 2 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-06-21T01:17:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-06-21T01:17:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)": 15 2024-06-21T01:17:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-06-21T01:17:03.725 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:17:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:03 smithi027 ceph-mon[84934]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:03.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:03 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/2259189867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:03.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:03 smithi027 ceph-mon[90235]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:03.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:03 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/2259189867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:03 smithi120 ceph-mon[67716]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:04.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:03 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/2259189867' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:04.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch upgrade status' 2024-06-21T01:17:04.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:04 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:04.921 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:04 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:05.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:04 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:05.944 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:05 smithi027 ceph-mon[84934]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:05.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:05 smithi027 ceph-mon[90235]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:06.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:05 smithi120 ceph-mon[67716]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0", 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "osd", 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "mon", 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr", 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw" 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-06-21T01:17:06.297 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "15/21 daemons upgraded", 2024-06-21T01:17:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed.", 2024-06-21T01:17:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": true 2024-06-21T01:17:06.298 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:17:06.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph health detail' 2024-06-21T01:17:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:07 smithi120 ceph-mon[67716]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:08.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:07 smithi120 ceph-mon[67716]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:08.140 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:07 smithi027 ceph-mon[84934]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:08.140 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:07 smithi027 ceph-mon[84934]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:08.140 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:07 smithi027 ceph-mon[90235]: from='client.64831 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:08.140 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:07 smithi027 ceph-mon[90235]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:08.728 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:08 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:17:08] "GET /metrics HTTP/1.1" 200 38391 "" "Prometheus/2.43.0" 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi027.temphd on host smithi027 failed. 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout: Upgrade daemon: iscsi.foo.smithi027.temphd: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi027.temphd ... 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:Creating ceph-iscsi config... 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/iscsi-gateway.cfg 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:Write file: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/iscsi.foo.smithi027.temphd/tcmu-runner-entrypoint.sh 2024-06-21T01:17:08.970 INFO:teuthology.orchestra.run.smithi027.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d47651940\x2d2f66\x2d11ef\x2dbc9d\x2dc7b262605968.slice/ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl start ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout:Traceback (most recent call last): 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: return _run_code(code, main_globals, None, 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: exec(code, run_globals) 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5618, in 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 5606, in main 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3096, in command_deploy_from 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3131, in _common_deploy 2024-06-21T01:17:08.971 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 3151, in _deploy_daemon_container 2024-06-21T01:17:08.972 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1087, in deploy_daemon 2024-06-21T01:17:08.972 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/__main__.py", line 1223, in deploy_daemon_units 2024-06-21T01:17:08.972 INFO:teuthology.orchestra.run.smithi027.stdout: File "/tmp/tmpnc17e2bk.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-06-21T01:17:08.972 INFO:teuthology.orchestra.run.smithi027.stdout:RuntimeError: Failed command: systemctl start ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd: Job for ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service failed because the control process exited with error code. 2024-06-21T01:17:08.972 INFO:teuthology.orchestra.run.smithi027.stdout:See "systemctl status ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" and "journalctl -xeu ceph-47651940-2f66-11ef-bc9d-c7b262605968@iscsi.foo.smithi027.temphd.service" for details. 2024-06-21T01:17:09.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-06-21T01:17:10.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:09 smithi120 ceph-mon[67716]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:10.118 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:09 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3606580625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:17:10.136 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:09 smithi027 ceph-mon[84934]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:10.136 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:09 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3606580625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:17:10.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:09 smithi027 ceph-mon[90235]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:10.136 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:09 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3606580625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-21T01:17:11.692 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-06-21T01:17:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:11 smithi120 ceph-mon[67716]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:12.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:11 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/1457011523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:11 smithi027 ceph-mon[84934]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:11 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/1457011523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:11 smithi027 ceph-mon[90235]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:12.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:11 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/1457011523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:12.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-06-21T01:17:12.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:12 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:12.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:12 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:13.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:12 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:14.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:13 smithi120 ceph-mon[67716]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:14.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:13 smithi027 ceph-mon[84934]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:14.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:13 smithi027 ceph-mon[90235]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:14.406 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.0.0-2541-gc65be810 (c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd) squid (rc)" 2024-06-21T01:17:14.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -e sha1=c65be8102fd0c62c21484f7d2c7085bf8cb6a8fd -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-06-21T01:17:15.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:14 smithi120 ceph-mon[67716]: from='client.? 172.21.15.27:0/3686197250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:15.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:14 smithi027 ceph-mon[84934]: from='client.? 172.21.15.27:0/3686197250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:15.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:14 smithi027 ceph-mon[90235]: from='client.? 172.21.15.27:0/3686197250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-21T01:17:16.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:15 smithi120 ceph-mon[67716]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:16.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:15 smithi027 ceph-mon[84934]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:16.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:15 smithi027 ceph-mon[90235]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:16.833 INFO:teuthology.orchestra.run.smithi027.stdout:osd 8 2m ago - 2024-06-21T01:17:17.556 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-21T01:17:17.566 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-06-21T01:17:17.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-06-21T01:17:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:17 smithi120 ceph-mon[67716]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:18.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:17 smithi120 ceph-mon[67716]: from='client.64855 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:18.148 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:17 smithi027 ceph-mon[84934]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:18.148 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:17 smithi027 ceph-mon[84934]: from='client.64855 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:18.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:17 smithi027 ceph-mon[90235]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:18.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:17 smithi027 ceph-mon[90235]: from='client.64855 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:18.946 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:18 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:17:18] "GET /metrics HTTP/1.1" 200 38391 "" "Prometheus/2.43.0" 2024-06-21T01:17:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:19 smithi027 ceph-mon[84934]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:20.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:19 smithi027 ceph-mon[84934]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:20.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:19 smithi027 ceph-mon[90235]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:20.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:19 smithi027 ceph-mon[90235]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:19 smithi120 ceph-mon[67716]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:20.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:19 smithi120 ceph-mon[67716]: from='mgr.25753 172.21.15.27:0/3376913698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout: "image": "quay.io/ceph/ceph", 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout: "registry": "quay.io", 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout: "bare_image": "ceph/ceph", 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout: "versions": [] 2024-06-21T01:17:20.538 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-21T01:17:21.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:20 smithi027 ceph-mon[84934]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:21.146 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:20 smithi027 ceph-mon[90235]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:21.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-06-21T01:17:21.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:20 smithi120 ceph-mon[67716]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:22.142 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:21 smithi027 ceph-mon[84934]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:22.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:21 smithi027 ceph-mon[90235]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:22.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:21 smithi120 ceph-mon[67716]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:22.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:22 smithi027 ceph-mon[84934]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:22.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:22 smithi027 ceph-mon[90235]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:23.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:22 smithi120 ceph-mon[67716]: from='client.64747 -' entity='client.iscsi.foo.smithi027.temphd' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-06-21T01:17:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "16.2.0", 2024-06-21T01:17:24.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:23 smithi027 ceph-mon[84934]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:24.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:23 smithi027 ceph-mon[90235]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:24.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:23 smithi120 ceph-mon[67716]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:24.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47651940-2f66-11ef-bc9d-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-06-21T01:17:25.104 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:24 smithi027 ceph-mon[84934]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:25.104 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:24 smithi027 ceph-mon[90235]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:25.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:24 smithi120 ceph-mon[67716]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:26.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:25 smithi027 ceph-mon[84934]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:26.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:25 smithi027 ceph-mon[90235]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:26.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:25 smithi120 ceph-mon[67716]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-06-21T01:17:27.251 INFO:teuthology.orchestra.run.smithi027.stdout: "v16.2.2", 2024-06-21T01:17:27.251 INFO:teuthology.orchestra.run.smithi027.stdout: "v16.2.2-20210505", 2024-06-21T01:17:27.865 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-06-21T01:17:27.874 INFO:tasks.cephadm:Teardown begin 2024-06-21T01:17:27.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-21T01:17:27.908 DEBUG:teuthology.orchestra.run.smithi120:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-21T01:17:27.941 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-06-21T01:17:27.941 DEBUG:teuthology.orchestra.run.smithi027:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-21T01:17:27.966 DEBUG:teuthology.orchestra.run.smithi120:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-21T01:17:27.999 INFO:tasks.cephadm:Stopping all daemons... 2024-06-21T01:17:27.999 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-06-21T01:17:27.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a 2024-06-21T01:17:28.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:27 smithi027 ceph-mon[84934]: from='client.64873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:28.033 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:27 smithi027 ceph-mon[84934]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:28.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:27 smithi027 ceph-mon[90235]: from='client.64873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:28.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:27 smithi027 ceph-mon[90235]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:28.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 systemd[1]: Stopping Ceph mon.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:27 smithi120 ceph-mon[67716]: from='client.64873 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-21T01:17:28.366 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:27 smithi120 ceph-mon[67716]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 267 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-06-21T01:17:28.639 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE Bus STOPPING 2024-06-21T01:17:28.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a[84925]: 2024-06-21T01:17:28.333+0000 7f56443f7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:28.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a[84925]: 2024-06-21T01:17:28.333+0000 7f56443f7640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-06-21T01:17:28.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 podman[130922]: 2024-06-21 01:17:28.374953171 +0000 UTC m=+0.164910736 container died 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 podman[130922]: 2024-06-21 01:17:28.758418081 +0000 UTC m=+0.548375646 container cleanup 65d2d766d07e2fbcad46b8fa9401ddb01a6803db51bb09be777f7cd2f0c5e721 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Jun 21 01:17:28 smithi027 bash[130922]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-a 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: ::ffff:172.21.15.120 - - [21/Jun/2024:01:17:28] "GET /metrics HTTP/1.1" 200 38394 "" "Prometheus/2.43.0" 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE Bus STOPPED 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE Bus STARTING 2024-06-21T01:17:28.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE Serving on http://:::9283 2024-06-21T01:17:28.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:28 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: [21/Jun/2024:01:17:28] ENGINE Bus STARTED 2024-06-21T01:17:29.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.a.service' 2024-06-21T01:17:29.281 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:29.281 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-06-21T01:17:29.281 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-06-21T01:17:29.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.c 2024-06-21T01:17:29.610 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:29 smithi027 systemd[1]: Stopping Ceph mon.c for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:29.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c[90231]: 2024-06-21T01:17:29.612+0000 7f3d6153e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:29.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:29 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c[90231]: 2024-06-21T01:17:29.612+0000 7f3d6153e640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-06-21T01:17:29.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:29 smithi027 podman[131055]: 2024-06-21 01:17:29.644500915 +0000 UTC m=+0.157623073 container died 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-06-21T01:17:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:30 smithi027 podman[131055]: 2024-06-21 01:17:30.038188569 +0000 UTC m=+0.551310718 container cleanup 7b7984c9e8ba8bf757292895e237d6e1269acc2afa5f219f48be504ab596ce34 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:17:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Jun 21 01:17:30 smithi027 bash[131055]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-c 2024-06-21T01:17:30.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.c.service' 2024-06-21T01:17:30.518 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:30.518 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-06-21T01:17:30.518 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-06-21T01:17:30.518 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.b 2024-06-21T01:17:30.818 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:30 smithi120 systemd[1]: Stopping Ceph mon.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:31.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b[67712]: 2024-06-21T01:17:30.820+0000 7f16a5cd2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:31.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b[67712]: 2024-06-21T01:17:30.820+0000 7f16a5cd2640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-06-21T01:17:31.116 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:30 smithi120 podman[101544]: 2024-06-21 01:17:30.922677232 +0000 UTC m=+0.203644759 container died 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-21T01:17:31.653 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:31 smithi120 podman[101544]: 2024-06-21 01:17:31.404726082 +0000 UTC m=+0.685693650 container cleanup 65c76c6e776b865fd0463873ae555ac40585d3d9b3b87990ce04401f1cbad91a (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:17:31.653 INFO:journalctl@ceph.mon.b.smithi120.stdout:Jun 21 01:17:31 smithi120 bash[101544]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mon-b 2024-06-21T01:17:31.825 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mon.b.service' 2024-06-21T01:17:31.884 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:31.884 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-06-21T01:17:31.884 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-06-21T01:17:31.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y 2024-06-21T01:17:32.216 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:31 smithi027 systemd[1]: Stopping Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:32.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:17:32.218+0000 7f3fa455e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:32.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y[63252]: 2024-06-21T01:17:32.218+0000 7f3fa455e640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-06-21T01:17:32.478 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 podman[131186]: 2024-06-21 01:17:32.249416255 +0000 UTC m=+0.151320812 container died 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS) 2024-06-21T01:17:32.768 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 podman[131186]: 2024-06-21 01:17:32.584364888 +0000 UTC m=+0.486269449 container cleanup 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618) 2024-06-21T01:17:32.768 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 bash[131186]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y 2024-06-21T01:17:33.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service' 2024-06-21T01:17:33.030 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:32 smithi027 podman[131201]: 2024-06-21 01:17:32.771535147 +0000 UTC m=+0.516882662 container remove 16f1f10d1db0ce54b70b4dffbc3a9e7dfff4a50303e726a8ef9933edf221d0db (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-y, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-06-21T01:17:33.030 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:33 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Deactivated successfully. 2024-06-21T01:17:33.031 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:33 smithi027 systemd[1]: Stopped Ceph mgr.y for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:17:33.031 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Jun 21 01:17:33 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.y.service: Consumed 45.645s CPU time. 2024-06-21T01:17:33.045 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:33.045 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-06-21T01:17:33.045 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-06-21T01:17:33.045 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x 2024-06-21T01:17:33.366 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:17:33 smithi120 systemd[1]: Stopping Ceph mgr.x for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:33.687 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:17:33 smithi120 podman[101664]: 2024-06-21 01:17:33.399194605 +0000 UTC m=+0.152926197 container died 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:17:34.034 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:17:33 smithi120 podman[101664]: 2024-06-21 01:17:33.785470854 +0000 UTC m=+0.539202518 container cleanup 277873bc752621d19869b64cc463f822641a3c6a0b4cafd310ba919801636783 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81) 2024-06-21T01:17:34.034 INFO:journalctl@ceph.mgr.x.smithi120.stdout:Jun 21 01:17:33 smithi120 bash[101664]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-mgr-x 2024-06-21T01:17:34.164 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@mgr.x.service' 2024-06-21T01:17:34.199 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:34.199 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-06-21T01:17:34.199 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-06-21T01:17:34.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.0 2024-06-21T01:17:34.478 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:34 smithi027 systemd[1]: Stopping Ceph osd.0 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:34.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:17:34.540+0000 7f68b450e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:34.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:17:34.540+0000 7f68b450e640 -1 osd.0 165 *** Got signal Terminated *** 2024-06-21T01:17:34.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:34 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0[97282]: 2024-06-21T01:17:34.540+0000 7f68b450e640 -1 osd.0 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:17:40.206 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:39 smithi027 podman[131307]: 2024-06-21 01:17:39.941671638 +0000 UTC m=+5.531908949 container died fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0) 2024-06-21T01:17:40.478 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:40 smithi027 podman[131307]: 2024-06-21 01:17:40.332950116 +0000 UTC m=+5.923187416 container cleanup fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:17:40.478 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:40 smithi027 bash[131307]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0 2024-06-21T01:17:40.978 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:40 smithi027 podman[131322]: 2024-06-21 01:17:40.651285205 +0000 UTC m=+0.710358738 container remove fb7aea189c7b85068a08c3cd5e761e502cffc36ba4f2657b6f59d8d7fedae8ec (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2) 2024-06-21T01:17:41.252 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:40 smithi027 podman[131412]: 2024-06-21 01:17:40.890135763 +0000 UTC m=+0.017963172 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:17:41.252 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:41 smithi027 podman[131412]: 2024-06-21 01:17:41.010798522 +0000 UTC m=+0.138625928 container create 3394fc68d99533f1af0f4dd3ffbfe4148028b802b89ae1420601e401ddfb8aa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-21T01:17:41.252 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:41 smithi027 podman[131412]: 2024-06-21 01:17:41.255154881 +0000 UTC m=+0.382982290 container init 3394fc68d99533f1af0f4dd3ffbfe4148028b802b89ae1420601e401ddfb8aa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True) 2024-06-21T01:17:41.523 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:41 smithi027 podman[131412]: 2024-06-21 01:17:41.259034694 +0000 UTC m=+0.386862106 container start 3394fc68d99533f1af0f4dd3ffbfe4148028b802b89ae1420601e401ddfb8aa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS) 2024-06-21T01:17:41.523 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:41 smithi027 podman[131412]: 2024-06-21 01:17:41.363490653 +0000 UTC m=+0.491318078 container attach 3394fc68d99533f1af0f4dd3ffbfe4148028b802b89ae1420601e401ddfb8aa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240618) 2024-06-21T01:17:41.838 INFO:journalctl@ceph.osd.0.smithi027.stdout:Jun 21 01:17:41 smithi027 podman[131412]: 2024-06-21 01:17:41.526271642 +0000 UTC m=+0.654099049 container died 3394fc68d99533f1af0f4dd3ffbfe4148028b802b89ae1420601e401ddfb8aa7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618) 2024-06-21T01:17:42.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.0.service' 2024-06-21T01:17:42.153 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:42.153 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-06-21T01:17:42.154 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-06-21T01:17:42.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.1 2024-06-21T01:17:42.364 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:42 smithi027 systemd[1]: Stopping Ceph osd.1 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:42.672 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:17:42.529+0000 7fc91a062640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:42.672 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:17:42.529+0000 7fc91a062640 -1 osd.1 165 *** Got signal Terminated *** 2024-06-21T01:17:42.672 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1[102404]: 2024-06-21T01:17:42.529+0000 7fc91a062640 -1 osd.1 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:17:48.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:47 smithi027 podman[131512]: 2024-06-21 01:17:47.929889828 +0000 UTC m=+5.523932718 container died eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618) 2024-06-21T01:17:48.611 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:48 smithi027 podman[131512]: 2024-06-21 01:17:48.32081748 +0000 UTC m=+5.914860369 container cleanup eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618) 2024-06-21T01:17:48.611 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:48 smithi027 bash[131512]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1 2024-06-21T01:17:48.953 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:48 smithi027 podman[131540]: 2024-06-21 01:17:48.613991261 +0000 UTC m=+0.678777506 container remove eb18d2365a8521a3a65b2cbecbb21941dd0dd6112ca0558c39372d0bff95b0f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-06-21T01:17:49.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:48 smithi027 podman[131628]: 2024-06-21 01:17:48.85883199 +0000 UTC m=+0.018101290 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:17:49.228 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:48 smithi027 podman[131628]: 2024-06-21 01:17:48.981451068 +0000 UTC m=+0.140720368 container create cb93c25598f2cd798f7eac5c2c97315e53fdf80035eedb85c91f0b3f07901adb (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0) 2024-06-21T01:17:49.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:49 smithi027 podman[131628]: 2024-06-21 01:17:49.193259226 +0000 UTC m=+0.352528531 container init cb93c25598f2cd798f7eac5c2c97315e53fdf80035eedb85c91f0b3f07901adb (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:17:49.229 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:49 smithi027 podman[131628]: 2024-06-21 01:17:49.196871382 +0000 UTC m=+0.356140681 container start cb93c25598f2cd798f7eac5c2c97315e53fdf80035eedb85c91f0b3f07901adb (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-06-21T01:17:49.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:49 smithi027 podman[131628]: 2024-06-21 01:17:49.267419595 +0000 UTC m=+0.426688900 container attach cb93c25598f2cd798f7eac5c2c97315e53fdf80035eedb85c91f0b3f07901adb (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:17:49.728 INFO:journalctl@ceph.osd.1.smithi027.stdout:Jun 21 01:17:49 smithi027 podman[131628]: 2024-06-21 01:17:49.467084875 +0000 UTC m=+0.626354168 container died cb93c25598f2cd798f7eac5c2c97315e53fdf80035eedb85c91f0b3f07901adb (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:17:50.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.1.service' 2024-06-21T01:17:50.130 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:50.130 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-06-21T01:17:50.130 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-06-21T01:17:50.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.2 2024-06-21T01:17:50.464 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:50 smithi027 systemd[1]: Stopping Ceph osd.2 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:50.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:17:50.465+0000 7fc83ff23640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:50.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:17:50.465+0000 7fc83ff23640 -1 osd.2 165 *** Got signal Terminated *** 2024-06-21T01:17:50.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:50 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2[108682]: 2024-06-21T01:17:50.465+0000 7fc83ff23640 -1 osd.2 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:17:56.228 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:55 smithi027 podman[131730]: 2024-06-21 01:17:55.934296936 +0000 UTC m=+5.569053943 container died c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-06-21T01:17:56.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:56 smithi027 podman[131730]: 2024-06-21 01:17:56.265864541 +0000 UTC m=+5.900621539 container cleanup c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-21T01:17:56.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:56 smithi027 bash[131730]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2 2024-06-21T01:17:56.728 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:56 smithi027 podman[131746]: 2024-06-21 01:17:56.500456235 +0000 UTC m=+0.560299495 container remove c604eb1e6396f8bfceac0b9061f030636e7006da969b197642fe7040caa5ef91 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, io.buildah.version=1.36.0, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-06-21T01:17:57.151 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:56 smithi027 podman[131831]: 2024-06-21 01:17:56.750763833 +0000 UTC m=+0.017545641 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:17:57.151 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:56 smithi027 podman[131831]: 2024-06-21 01:17:56.868338154 +0000 UTC m=+0.135119950 container create 653c6f1c06ba38fe9b91fdbeb8a1d6fd8e6e8d25f3af902018036046894f9493 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-release-c65be81) 2024-06-21T01:17:57.151 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:57 smithi027 podman[131831]: 2024-06-21 01:17:57.070884948 +0000 UTC m=+0.337666758 container init 653c6f1c06ba38fe9b91fdbeb8a1d6fd8e6e8d25f3af902018036046894f9493 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:17:57.151 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:57 smithi027 podman[131831]: 2024-06-21 01:17:57.074187756 +0000 UTC m=+0.340969556 container start 653c6f1c06ba38fe9b91fdbeb8a1d6fd8e6e8d25f3af902018036046894f9493 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-06-21T01:17:57.151 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:57 smithi027 podman[131831]: 2024-06-21 01:17:57.154029808 +0000 UTC m=+0.420811608 container attach 653c6f1c06ba38fe9b91fdbeb8a1d6fd8e6e8d25f3af902018036046894f9493 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0) 2024-06-21T01:17:57.478 INFO:journalctl@ceph.osd.2.smithi027.stdout:Jun 21 01:17:57 smithi027 podman[131831]: 2024-06-21 01:17:57.33820046 +0000 UTC m=+0.604982264 container died 653c6f1c06ba38fe9b91fdbeb8a1d6fd8e6e8d25f3af902018036046894f9493 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=) 2024-06-21T01:17:57.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.2.service' 2024-06-21T01:17:57.966 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:17:57.967 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-06-21T01:17:57.967 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-06-21T01:17:57.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.3 2024-06-21T01:17:58.228 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:17:58 smithi027 systemd[1]: Stopping Ceph osd.3 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:17:58.728 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:17:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:17:58.344+0000 7ff297315640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:17:58.728 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:17:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:17:58.344+0000 7ff297315640 -1 osd.3 165 *** Got signal Terminated *** 2024-06-21T01:17:58.728 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:17:58 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:17:58.344+0000 7ff297315640 -1 osd.3 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:18:01.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:00 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:00.942+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:01.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:01 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:18:01.158+0000 7ff28f923640 -1 osd.3 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:35.439020+0000 front 2024-06-21T01:17:35.438947+0000 (oldest deadline 2024-06-21T01:18:00.738878+0000) 2024-06-21T01:18:02.116 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:01.938+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:02.116 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:01 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:01.930+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:02.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:02 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:18:02.112+0000 7ff28f923640 -1 osd.3 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:35.439020+0000 front 2024-06-21T01:17:35.438947+0000 (oldest deadline 2024-06-21T01:18:00.738878+0000) 2024-06-21T01:18:02.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:02.412+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:03.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:02.986+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:03.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:02.899+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:03.366 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:02.950Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-06-21T01:18:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:02.952Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-06-21T01:18:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:02.952Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-06-21T01:18:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:02.954Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-06-21T01:18:03.367 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:02 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:02.954Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-06-21T01:18:03.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:03 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3[115219]: 2024-06-21T01:18:03.160+0000 7ff28f923640 -1 osd.3 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:35.439020+0000 front 2024-06-21T01:17:35.438947+0000 (oldest deadline 2024-06-21T01:18:00.738878+0000) 2024-06-21T01:18:03.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:03.396+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:04.050 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:03 smithi027 podman[131932]: 2024-06-21 01:18:03.753640071 +0000 UTC m=+5.549022541 container died 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-06-21T01:18:04.334 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 podman[131932]: 2024-06-21 01:18:04.152711123 +0000 UTC m=+5.948093594 container cleanup 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, org.label-schema.build-date=20240618, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-06-21T01:18:04.334 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 bash[131932]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3 2024-06-21T01:18:04.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:03.937+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:04.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:03 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:03.940+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:04.728 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 podman[131959]: 2024-06-21 01:18:04.337452333 +0000 UTC m=+0.580060060 container remove 6e5b1a5a8360cb9936893ac1596006829026106e373c6c0f7fc7f2f65f0f7b86 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:18:04.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:04.379+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:05.071 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 podman[132051]: 2024-06-21 01:18:04.779940085 +0000 UTC m=+0.104261215 container create ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS) 2024-06-21T01:18:05.071 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 podman[132051]: 2024-06-21 01:18:04.693777736 +0000 UTC m=+0.018098879 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:18:05.071 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:04 smithi027 podman[132051]: 2024-06-21 01:18:04.999196934 +0000 UTC m=+0.323518076 container init ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-21T01:18:05.071 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:05 smithi027 podman[132051]: 2024-06-21 01:18:05.00275064 +0000 UTC m=+0.327071777 container start ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:05.071 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:05 smithi027 podman[132051]: 2024-06-21 01:18:05.073923664 +0000 UTC m=+0.398244799 container attach ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-21T01:18:05.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:04.905+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:05.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:04.904+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:05.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:04 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:04.972+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:05.478 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:05 smithi027 podman[132051]: 2024-06-21 01:18:05.278789639 +0000 UTC m=+0.603110772 container died ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:05.833 INFO:journalctl@ceph.osd.3.smithi027.stdout:Jun 21 01:18:05 smithi027 podman[132074]: 2024-06-21 01:18:05.813363676 +0000 UTC m=+0.527684118 container remove ed1d193ab8bf3d2f6276d41bca1f48fe4870e834ce9e862429d9d38725808303 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-06-21T01:18:05.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.3.service' 2024-06-21T01:18:05.866 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:05.405+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:05.884 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:05.884 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-06-21T01:18:05.884 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-06-21T01:18:05.884 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.4 2024-06-21T01:18:06.285 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:05.906+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:06.286 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:05.944+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:06.286 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:05 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:05.902+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:06.286 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:05 smithi120 systemd[1]: Stopping Ceph osd.4 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:06.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:06.359+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:06.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:06.287+0000 7f177a6f5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:18:06.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:06.287+0000 7f177a6f5640 -1 osd.4 165 *** Got signal Terminated *** 2024-06-21T01:18:06.616 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:06.287+0000 7f177a6f5640 -1 osd.4 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:18:07.349 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:06.911+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:07.350 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:06.935+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:07.350 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:06 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:06.919+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:07.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:07.351+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:08.359 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:07.890+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:08.359 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:07.934+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:08.360 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:07 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:07.928+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:08.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:08.361+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:09.116 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:08.848+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:09.116 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:08.907+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:09.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:08.982+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:09.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:08 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:08.982+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:09.615 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:09.321+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:10.323 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:09.877+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:10.323 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:09.877+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:46.589214+0000 front 2024-06-21T01:17:46.589310+0000 (oldest deadline 2024-06-21T01:18:08.889082+0000) 2024-06-21T01:18:10.324 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:09.915+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:10.324 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:10.021+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:10.324 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:10.021+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:10.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:10.325+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:10.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:10.325+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:11.354 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:10.952+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:11.354 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:10.869+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.988460+0000 front 2024-06-21T01:17:38.988202+0000 (oldest deadline 2024-06-21T01:18:00.688318+0000) 2024-06-21T01:18:11.354 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4[78399]: 2024-06-21T01:18:10.869+0000 7f1776d0b640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:46.589214+0000 front 2024-06-21T01:17:46.589310+0000 (oldest deadline 2024-06-21T01:18:08.889082+0000) 2024-06-21T01:18:11.354 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:10.995+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:11.355 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:10 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:10.995+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:11.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:11.355+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:11.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:11.355+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:12.052 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:11 smithi120 podman[101800]: 2024-06-21 01:18:11.802204869 +0000 UTC m=+5.711813605 container died c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD) 2024-06-21T01:18:12.053 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:11.988+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:12.053 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:11.972+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:12.053 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:11 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:11.972+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:12.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:12 smithi120 podman[101800]: 2024-06-21 01:18:12.19676217 +0000 UTC m=+6.106370974 container cleanup c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-06-21T01:18:12.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:12 smithi120 bash[101800]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4 2024-06-21T01:18:12.832 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:12 smithi120 podman[101817]: 2024-06-21 01:18:12.473406521 +0000 UTC m=+0.663051765 container remove c47340426cbf1ec1617006b256f469fee7f5b32ff6f17ba9eaa82cad18e5e03d (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:12.833 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:12.397+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:12.833 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:12.397+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:13.091 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:12.941+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:13.092 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:12 smithi120 podman[101908]: 2024-06-21 01:18:12.73852479 +0000 UTC m=+0.018481225 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:18:13.092 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:12 smithi120 podman[101908]: 2024-06-21 01:18:12.875020977 +0000 UTC m=+0.154977454 container create 9701a7c0900af86ecbe422543f195d7bffc86a13ced4fbaf2cce5baeddd28047 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240618) 2024-06-21T01:18:13.092 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:12.929+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:13.092 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:12 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:12.929+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:13.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:13 smithi120 podman[101908]: 2024-06-21 01:18:13.094420844 +0000 UTC m=+0.374377278 container init 9701a7c0900af86ecbe422543f195d7bffc86a13ced4fbaf2cce5baeddd28047 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-06-21T01:18:13.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:13 smithi120 podman[101908]: 2024-06-21 01:18:13.098115012 +0000 UTC m=+0.378071451 container start 9701a7c0900af86ecbe422543f195d7bffc86a13ced4fbaf2cce5baeddd28047 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:18:13.366 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:13 smithi120 podman[101908]: 2024-06-21 01:18:13.177667015 +0000 UTC m=+0.457623466 container attach 9701a7c0900af86ecbe422543f195d7bffc86a13ced4fbaf2cce5baeddd28047 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618) 2024-06-21T01:18:13.637 INFO:journalctl@ceph.osd.4.smithi120.stdout:Jun 21 01:18:13 smithi120 podman[101908]: 2024-06-21 01:18:13.369521517 +0000 UTC m=+0.649477954 container died 9701a7c0900af86ecbe422543f195d7bffc86a13ced4fbaf2cce5baeddd28047 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.build-date=20240618) 2024-06-21T01:18:13.637 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:13.383+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:13.637 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:13.383+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:13.907 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.4.service' 2024-06-21T01:18:13.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:13.945 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-06-21T01:18:13.945 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-06-21T01:18:13.945 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.5 2024-06-21T01:18:14.274 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:13.938+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:14.274 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:13.938+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:13.893+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:13 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:13.893+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 systemd[1]: Stopping Ceph osd.5 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:14.276+0000 7f51f42b0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:14.276+0000 7f51f42b0640 -1 osd.5 165 *** Got signal Terminated *** 2024-06-21T01:18:14.275 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:14.276+0000 7f51f42b0640 -1 osd.5 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:18:14.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:14.336+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:14.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:14.336+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:15.286 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:14.882+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:15.286 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:14.882+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:15.287 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:14.976+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:15.287 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:14 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:14.976+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:15.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:15.288+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:15.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:15.288+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:16.241 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:15.918+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:16.241 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:15.918+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:16.242 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:15.940+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:16.242 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:15 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:15.940+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:16.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:16.243+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:16.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:16.243+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:17.244 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:16.959+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:17.244 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:16.959+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:17.244 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:16.948+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:17.244 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:16.948+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:17.244 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:16 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:16.948+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:17.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:17.245+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:17.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:17.245+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:18.218 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:17.966+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:18.219 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:17.966+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:18.219 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:17.940+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:18.219 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:17.940+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:18.219 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:17 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:17.940+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:18.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:18.220+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:18.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:18.220+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:19.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:19.003+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.048437+0000 front 2024-06-21T01:17:39.048274+0000 (oldest deadline 2024-06-21T01:18:01.348358+0000) 2024-06-21T01:18:19.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5[83014]: 2024-06-21T01:18:19.003+0000 7f51f08c6640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:45.349466+0000 front 2024-06-21T01:17:45.349543+0000 (oldest deadline 2024-06-21T01:18:13.550066+0000) 2024-06-21T01:18:19.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:18.982+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:19.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:18.982+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:19.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:18 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:18.982+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:19.367 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:19.206+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:19.367 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:19.206+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:19.866 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:19 smithi120 podman[102019]: 2024-06-21 01:18:19.589547146 +0000 UTC m=+5.412829299 container died a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-release-c65be81) 2024-06-21T01:18:20.164 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:19.967+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:20.164 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:19.967+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:20.164 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:19 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:19.967+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:20.165 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102019]: 2024-06-21 01:18:20.043396741 +0000 UTC m=+5.866678888 container cleanup a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD) 2024-06-21T01:18:20.165 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 bash[102019]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5 2024-06-21T01:18:20.616 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102035]: 2024-06-21 01:18:20.345495497 +0000 UTC m=+0.754658518 container remove a8c0ce0059e8819cbae3a9b5b5a53e6215918d12bcb24802adbda4a1dc2b9eb1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5, RELEASE=squid-release-c65be81, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD) 2024-06-21T01:18:20.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:20.166+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:20.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:20.166+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:20.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:20.166+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:20.938 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102125]: 2024-06-21 01:18:20.580088236 +0000 UTC m=+0.016841996 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:18:20.938 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102125]: 2024-06-21 01:18:20.679864958 +0000 UTC m=+0.116618748 container create ca0b8f00ece1b022db31b1bc223eab24b10bfb3f57e8d286f4fbc93b39e66886 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, RELEASE=squid-release-c65be81, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-06-21T01:18:20.938 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102125]: 2024-06-21 01:18:20.857595458 +0000 UTC m=+0.294349217 container init ca0b8f00ece1b022db31b1bc223eab24b10bfb3f57e8d286f4fbc93b39e66886 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-release-c65be81, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:18:20.938 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102125]: 2024-06-21 01:18:20.861174482 +0000 UTC m=+0.297928230 container start ca0b8f00ece1b022db31b1bc223eab24b10bfb3f57e8d286f4fbc93b39e66886 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:20.938 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:20 smithi120 podman[102125]: 2024-06-21 01:18:20.940666946 +0000 UTC m=+0.377420705 container attach ca0b8f00ece1b022db31b1bc223eab24b10bfb3f57e8d286f4fbc93b39e66886 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:18:21.366 INFO:journalctl@ceph.osd.5.smithi120.stdout:Jun 21 01:18:21 smithi120 podman[102125]: 2024-06-21 01:18:21.130197397 +0000 UTC m=+0.566951145 container died ca0b8f00ece1b022db31b1bc223eab24b10bfb3f57e8d286f4fbc93b39e66886 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240618, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-06-21T01:18:21.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:21.158+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:21.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:21.158+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:21.367 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:21.158+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:21.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:20.983+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:21.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:20.983+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:21.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:20 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:20.983+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:21.756 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.5.service' 2024-06-21T01:18:21.798 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:21.798 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-06-21T01:18:21.798 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-06-21T01:18:21.798 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.6 2024-06-21T01:18:22.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:21 smithi120 systemd[1]: Stopping Ceph osd.6 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:22.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:21.971+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:22.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:21.971+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:22.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:21 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:21.971+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:22.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:22.180+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:22.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:22.180+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:22.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:22.180+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:22.616 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.156+0000 7f9b0c60e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:18:22.616 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.156+0000 7f9b0c60e640 -1 osd.6 165 *** Got signal Terminated *** 2024-06-21T01:18:22.617 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.156+0000 7f9b0c60e640 -1 osd.6 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.966+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.966+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:22 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:22.966+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:23.165+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:23.165+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:23.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:23 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:23.165+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:24.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:24.010+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:24.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:24.010+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:24.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:24.010+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:24.116 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:24.010+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:57.930926+0000 front 2024-06-21T01:17:57.930963+0000 (oldest deadline 2024-06-21T01:18:23.830926+0000) 2024-06-21T01:18:24.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:24.176+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:24.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:24.176+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:24.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:24 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:24.176+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:25.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:25.138+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:25.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:25.138+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:25.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:25.138+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:25.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:25.025+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:25.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:25.025+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:25.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:25.025+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:25.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:25 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:25.025+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:57.930926+0000 front 2024-06-21T01:17:57.930963+0000 (oldest deadline 2024-06-21T01:18:23.830926+0000) 2024-06-21T01:18:26.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:26.112+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:26.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:26.112+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:26.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:26.112+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:26.370 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:26.112+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:26.370 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:26.074+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:26.370 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:26.074+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:26.371 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:26.074+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:26.371 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:26 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:26.074+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:57.930926+0000 front 2024-06-21T01:17:57.930963+0000 (oldest deadline 2024-06-21T01:18:23.830926+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:27.121+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:27.121+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:27.121+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:27.121+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:27.026+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:38.629020+0000 front 2024-06-21T01:17:38.629033+0000 (oldest deadline 2024-06-21T01:18:04.528906+0000) 2024-06-21T01:18:27.366 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:27.026+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:44.529391+0000 front 2024-06-21T01:17:44.529400+0000 (oldest deadline 2024-06-21T01:18:08.629343+0000) 2024-06-21T01:18:27.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:27.026+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.530297+0000 front 2024-06-21T01:17:54.530355+0000 (oldest deadline 2024-06-21T01:18:16.830217+0000) 2024-06-21T01:18:27.367 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:27 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6[87645]: 2024-06-21T01:18:27.026+0000 7f9b04c1c640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:57.930926+0000 front 2024-06-21T01:17:57.930963+0000 (oldest deadline 2024-06-21T01:18:23.830926+0000) 2024-06-21T01:18:27.979 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:27 smithi120 podman[102227]: 2024-06-21 01:18:27.674881209 +0000 UTC m=+5.652727314 container died cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, ceph=True, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-06-21T01:18:28.304 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102227]: 2024-06-21 01:18:28.081280714 +0000 UTC m=+6.059126818 container cleanup cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240618, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-21T01:18:28.305 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 bash[102227]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6 2024-06-21T01:18:28.305 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:28.171+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:28.305 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:28.171+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:28.305 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:28.171+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:28.305 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:28 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:28.171+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:28.616 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102242]: 2024-06-21 01:18:28.307659157 +0000 UTC m=+0.626295069 container remove cb59bc5cec74be8689b60669599131872030afff767d62c8f356a6c2d23e51f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-release-c65be81, ceph=True, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-21T01:18:28.891 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102330]: 2024-06-21 01:18:28.53701871 +0000 UTC m=+0.017417649 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:18:28.892 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102330]: 2024-06-21 01:18:28.650250949 +0000 UTC m=+0.130649930 container create 54dfb91ea9e873839ea3bb1b63dba12f8e03e70f995c9efc707a3cb4792a808f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, ceph=True) 2024-06-21T01:18:28.892 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102330]: 2024-06-21 01:18:28.836379856 +0000 UTC m=+0.316778796 container init 54dfb91ea9e873839ea3bb1b63dba12f8e03e70f995c9efc707a3cb4792a808f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240618, ceph=True, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:28.892 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102330]: 2024-06-21 01:18:28.839830487 +0000 UTC m=+0.320229422 container start 54dfb91ea9e873839ea3bb1b63dba12f8e03e70f995c9efc707a3cb4792a808f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240618, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-06-21T01:18:28.892 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:28 smithi120 podman[102330]: 2024-06-21 01:18:28.894633567 +0000 UTC m=+0.375032512 container attach 54dfb91ea9e873839ea3bb1b63dba12f8e03e70f995c9efc707a3cb4792a808f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-release-c65be81, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-21T01:18:29.187 INFO:journalctl@ceph.osd.6.smithi120.stdout:Jun 21 01:18:29 smithi120 podman[102330]: 2024-06-21 01:18:29.095871806 +0000 UTC m=+0.576270742 container died 54dfb91ea9e873839ea3bb1b63dba12f8e03e70f995c9efc707a3cb4792a808f (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, ceph=True, RELEASE=squid-release-c65be81, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-21T01:18:29.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:29.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:29.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:29.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:29.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:29.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:29.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:29 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:29.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:29.676 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.6.service' 2024-06-21T01:18:29.713 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:29.714 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-06-21T01:18:29.714 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-06-21T01:18:29.714 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.7 2024-06-21T01:18:30.040 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:29 smithi120 systemd[1]: Stopping Ceph osd.7 for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.042+0000 7f3d5a9a6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.042+0000 7f3d5a9a6640 -1 osd.7 165 *** Got signal Terminated *** 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.042+0000 7f3d5a9a6640 -1 osd.7 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.186+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.186+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.186+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:30.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:30 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:30.186+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:31.516 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:31.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:31.516 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:31.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:31.516 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:31.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:31.516 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:31 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:31.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:32.438 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:32.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:32.439 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:32.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:32.439 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:32.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:32.439 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:32 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:32.190+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:33.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:33.191+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:33.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:33.191+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:33.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:33.191+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:33.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:33 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:33.191+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:34.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:34.215+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-06-21T01:17:39.346298+0000 front 2024-06-21T01:17:39.346412+0000 (oldest deadline 2024-06-21T01:18:02.246296+0000) 2024-06-21T01:18:34.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:34.215+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-06-21T01:17:42.246821+0000 front 2024-06-21T01:17:42.246858+0000 (oldest deadline 2024-06-21T01:18:09.847205+0000) 2024-06-21T01:18:34.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:34.215+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-06-21T01:17:54.348920+0000 front 2024-06-21T01:17:54.348992+0000 (oldest deadline 2024-06-21T01:18:19.648939+0000) 2024-06-21T01:18:34.616 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:34 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7[92588]: 2024-06-21T01:18:34.215+0000 7f3d56fbc640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-06-21T01:17:59.649463+0000 front 2024-06-21T01:17:59.649477+0000 (oldest deadline 2024-06-21T01:18:25.549368+0000) 2024-06-21T01:18:35.716 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:35 smithi120 podman[102433]: 2024-06-21 01:18:35.467595182 +0000 UTC m=+5.530942560 container died 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81) 2024-06-21T01:18:36.083 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:35 smithi120 podman[102433]: 2024-06-21 01:18:35.909615481 +0000 UTC m=+5.972962861 container cleanup 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, GIT_BRANCH=HEAD) 2024-06-21T01:18:36.084 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:35 smithi120 bash[102433]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7 2024-06-21T01:18:36.366 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102464]: 2024-06-21 01:18:36.086459679 +0000 UTC m=+0.610507113 container remove 5098c861dd5c556152c8c82c62fcb3058bdfaebf18cba370767484eacd8f1139 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-06-21T01:18:36.837 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102552]: 2024-06-21 01:18:36.429005939 +0000 UTC m=+0.017226947 image pull e9bfc9e474e5aa1c9fe2590308cf585a9ebc8ed3f9cf85835780aeab9836d1d3 quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0 2024-06-21T01:18:36.837 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102552]: 2024-06-21 01:18:36.545785319 +0000 UTC m=+0.134006365 container create 0232f346c353c4a0541d8beb0ab593ea36ca75ae21c50f089055cca56f7b3b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-06-21T01:18:36.837 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102552]: 2024-06-21 01:18:36.756609427 +0000 UTC m=+0.344830444 container init 0232f346c353c4a0541d8beb0ab593ea36ca75ae21c50f089055cca56f7b3b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240618, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-release-c65be81, maintainer=Guillaume Abrioux ) 2024-06-21T01:18:36.837 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102552]: 2024-06-21 01:18:36.75985394 +0000 UTC m=+0.348074943 container start 0232f346c353c4a0541d8beb0ab593ea36ca75ae21c50f089055cca56f7b3b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0, RELEASE=squid-release-c65be81, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-06-21T01:18:36.837 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:36 smithi120 podman[102552]: 2024-06-21 01:18:36.83984031 +0000 UTC m=+0.428061325 container attach 0232f346c353c4a0541d8beb0ab593ea36ca75ae21c50f089055cca56f7b3b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-release-c65be81, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240618, org.label-schema.schema-version=1.0) 2024-06-21T01:18:37.116 INFO:journalctl@ceph.osd.7.smithi120.stdout:Jun 21 01:18:37 smithi120 podman[102552]: 2024-06-21 01:18:37.025422823 +0000 UTC m=+0.613643838 container died 0232f346c353c4a0541d8beb0ab593ea36ca75ae21c50f089055cca56f7b3b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:bba7e76a3887fd02e4f2e2ed3d352a62ae2630a800f64fb0e303a73f215a3fe0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240618, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-release-c65be81) 2024-06-21T01:18:37.675 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@osd.7.service' 2024-06-21T01:18:37.716 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:37.716 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-06-21T01:18:37.716 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-06-21T01:18:37.716 DEBUG:teuthology.orchestra.run.smithi120:> sudo systemctl stop ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a 2024-06-21T01:18:38.036 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:37 smithi120 systemd[1]: Stopping Ceph prometheus.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:38.328 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:18:38.328 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.039Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.040Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.040Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a[63907]: ts=2024-06-21T01:18:38.040Z caller=main.go:1147 level=info msg="See you next time!" 2024-06-21T01:18:38.329 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 podman[102653]: 2024-06-21 01:18:38.049424212 +0000 UTC m=+0.114861315 container died a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:38.615 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 podman[102653]: 2024-06-21 01:18:38.505886265 +0000 UTC m=+0.571323419 container cleanup a0fa635265143a595c3866e1d924a37f12f856f789740964d37f0a768519f1f1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:38.616 INFO:journalctl@ceph.prometheus.a.smithi120.stdout:Jun 21 01:18:38 smithi120 bash[102653]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-prometheus-a 2024-06-21T01:18:38.928 DEBUG:teuthology.orchestra.run.smithi120:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47651940-2f66-11ef-bc9d-c7b262605968@prometheus.a.service' 2024-06-21T01:18:38.961 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-21T01:18:38.961 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-06-21T01:18:38.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47651940-2f66-11ef-bc9d-c7b262605968 --force --keep-logs 2024-06-21T01:18:40.805 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:40 smithi027 systemd[1]: Stopping Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:41.086 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:40 smithi027 podman[132428]: 2024-06-21 01:18:40.807655811 +0000 UTC m=+0.134739014 container died 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:41.428 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 podman[132428]: 2024-06-21 01:18:41.196628296 +0000 UTC m=+0.523711504 container cleanup 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:41.429 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 bash[132428]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a 2024-06-21T01:18:41.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 podman[132442]: 2024-06-21 01:18:41.431505877 +0000 UTC m=+0.627391562 container remove 83944b7f409ef3cea35b487707346afb6ee1f2cc49e2750f6d4dd5e640adfff0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:41.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-06-21T01:18:41.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-06-21T01:18:41.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 systemd[1]: Stopped Ceph node-exporter.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:18:41.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Jun 21 01:18:41 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.a.service: Consumed 3.291s CPU time. 2024-06-21T01:18:42.032 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:41 smithi027 systemd[1]: Stopping Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:18:42.341 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a[80145]: ts=2024-06-21T01:18:42.034Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-06-21T01:18:42.341 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 podman[132558]: 2024-06-21 01:18:42.0758166 +0000 UTC m=+0.151762714 container died 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:42.691 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 podman[132558]: 2024-06-21 01:18:42.443518218 +0000 UTC m=+0.519464333 container cleanup 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:42.691 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 bash[132558]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a 2024-06-21T01:18:42.979 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 podman[132571]: 2024-06-21 01:18:42.694656556 +0000 UTC m=+0.609795112 container remove 32c04beeb6472a9f15ae9aff3c8b91c01012e57e1939b79a311e0cedc5c349f4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-06-21T01:18:43.042 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 podman[132571]: 2024-06-21 01:18:42.77026592 +0000 UTC m=+0.685404479 volume remove 1dc9e94ba6dabedf6c386af78097fedfe68345bad4101323988db9aa4744900f 2024-06-21T01:18:43.043 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-06-21T01:18:43.043 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 systemd[1]: Stopped Ceph alertmanager.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:18:43.043 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Jun 21 01:18:42 smithi027 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@alertmanager.a.service: Consumed 1.549s CPU time. 2024-06-21T01:19:06.029 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47651940-2f66-11ef-bc9d-c7b262605968 --force --keep-logs 2024-06-21T01:19:07.789 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:07 smithi120 systemd[1]: Stopping Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:19:08.047 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:07 smithi120 podman[103060]: 2024-06-21 01:19:07.8021885 +0000 UTC m=+0.110945188 container died b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:19:08.365 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 podman[103060]: 2024-06-21 01:19:08.25050247 +0000 UTC m=+0.559259199 container cleanup b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:19:08.366 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 bash[103060]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b 2024-06-21T01:19:08.838 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 podman[103075]: 2024-06-21 01:19:08.535136612 +0000 UTC m=+0.734534915 container remove b727660fc48b49150a80930a99d7bb943268456b9422d2327865be8a43bffc8f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-06-21T01:19:08.839 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-06-21T01:19:08.839 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-06-21T01:19:08.839 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 systemd[1]: Stopped Ceph node-exporter.b for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:19:08.839 INFO:journalctl@ceph.node-exporter.b.smithi120.stdout:Jun 21 01:19:08 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@node-exporter.b.service: Consumed 3.070s CPU time. 2024-06-21T01:19:09.116 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:08 smithi120 systemd[1]: Stopping Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968... 2024-06-21T01:19:09.116 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=server t=2024-06-21T01:19:09.080259814Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-06-21T01:19:09.116 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=ticker t=2024-06-21T01:19:09.080600935Z level=info msg=stopped last_tick=2024-06-21T01:19:00Z 2024-06-21T01:19:09.116 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a[66102]: logger=tracing t=2024-06-21T01:19:09.080638712Z level=info msg="Closing tracing" 2024-06-21T01:19:09.116 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 podman[103180]: 2024-06-21 01:19:09.090434063 +0000 UTC m=+0.106631245 container died 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:19:09.616 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 podman[103180]: 2024-06-21 01:19:09.405190186 +0000 UTC m=+0.421387368 container cleanup 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:19:09.616 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 bash[103180]: ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a 2024-06-21T01:19:09.917 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 podman[103194]: 2024-06-21 01:19:09.623250013 +0000 UTC m=+0.533442649 container remove 8bdfdc7ff2cd2839844a232339b344046bcfcfacd8ca5a3a3ee7fed22395025d (image=quay.io/ceph/grafana:9.4.12, name=ceph-47651940-2f66-11ef-bc9d-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-06-21T01:19:09.917 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Deactivated successfully. 2024-06-21T01:19:09.917 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 systemd[1]: Stopped Ceph grafana.a for 47651940-2f66-11ef-bc9d-c7b262605968. 2024-06-21T01:19:09.917 INFO:journalctl@ceph.grafana.a.smithi120.stdout:Jun 21 01:19:09 smithi120 systemd[1]: ceph-47651940-2f66-11ef-bc9d-c7b262605968@grafana.a.service: Consumed 10.689s CPU time. 2024-06-21T01:19:21.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-21T01:19:21.442 DEBUG:teuthology.orchestra.run.smithi120:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-21T01:19:21.469 INFO:tasks.cephadm:Archiving crash dumps... 2024-06-21T01:19:21.471 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi027/crash 2024-06-21T01:19:21.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash -- . 2024-06-21T01:19:21.524 INFO:teuthology.orchestra.run.smithi027.stderr:tar: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash: Cannot open: No such file or directory 2024-06-21T01:19:21.524 INFO:teuthology.orchestra.run.smithi027.stderr:tar: Error is not recoverable: exiting now 2024-06-21T01:19:21.526 DEBUG:teuthology.misc:Transferring archived files from smithi120:/var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi120/crash 2024-06-21T01:19:21.527 DEBUG:teuthology.orchestra.run.smithi120:> sudo tar c -f - -C /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash -- . 2024-06-21T01:19:21.561 INFO:teuthology.orchestra.run.smithi120.stderr:tar: /var/lib/ceph/47651940-2f66-11ef-bc9d-c7b262605968/crash: Cannot open: No such file or directory 2024-06-21T01:19:21.561 INFO:teuthology.orchestra.run.smithi120.stderr:tar: Error is not recoverable: exiting now 2024-06-21T01:19:21.563 INFO:tasks.cephadm:Checking cluster log for badness... 2024-06-21T01:19:21.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-21T01:19:21.597 INFO:teuthology.orchestra.run.smithi027.stdout:2024-06-21T00:54:12.634708+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-21T01:19:21.597 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-06-21T01:19:21.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[SEC\]' /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-21T01:19:21.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[ERR\]' /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-21T01:19:21.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[WRN\]' /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-21T01:19:21.820 INFO:teuthology.orchestra.run.smithi027.stdout:2024-06-21T00:54:12.634708+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-06-21T01:19:21.820 INFO:tasks.cephadm:Compressing logs... 2024-06-21T01:19:21.820 DEBUG:teuthology.orchestra.run.smithi027:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-21T01:19:21.864 DEBUG:teuthology.orchestra.run.smithi120:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-21T01:19:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-06-21T01:19:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-21T01:19:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log 2024-06-21T01:19:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.2.log 2024-06-21T01:19:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log 2024-06-21T01:19:21.889 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.3.log 2024-06-21T01:19:21.889 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.2.log: /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.1.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:: No such file or directory 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.7.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.5.log 2024-06-21T01:19:21.892 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.4.log 2024-06-21T01:19:21.893 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.7.log: /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.x.log 2024-06-21T01:19:21.893 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.6.log 2024-06-21T01:19:21.893 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log 2024-06-21T01:19:21.894 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.6.log: 89.0% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log.gz 2024-06-21T01:19:21.894 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.3.log: /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.y.log 2024-06-21T01:19:21.894 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.y.log: 93.9% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.log.gz 2024-06-21T01:19:21.895 INFO:teuthology.orchestra.run.smithi120.stderr: 91.1% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log.gz 2024-06-21T01:19:21.896 INFO:teuthology.orchestra.run.smithi120.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi120.bfbgme.log 2024-06-21T01:19:21.897 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log 2024-06-21T01:19:21.897 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi120.bfbgme.log: 75.7% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi120.bfbgme.log.gz 2024-06-21T01:19:21.898 INFO:teuthology.orchestra.run.smithi120.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.b.log 2024-06-21T01:19:21.898 INFO:teuthology.orchestra.run.smithi120.stderr: 89.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log.gz 2024-06-21T01:19:21.900 INFO:teuthology.orchestra.run.smithi027.stderr: 94.4% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.audit.log.gz 2024-06-21T01:19:21.900 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.0.log 2024-06-21T01:19:21.901 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.c.log 2024-06-21T01:19:21.901 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.a.log 2024-06-21T01:19:21.904 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.c.log: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-21T01:19:21.904 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log 2024-06-21T01:19:21.915 INFO:teuthology.orchestra.run.smithi120.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log: /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log.gz 2024-06-21T01:19:21.917 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/tcmu-runner.log 2024-06-21T01:19:21.920 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log: 93.8% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph.cephadm.log.gz 2024-06-21T01:19:21.921 INFO:teuthology.orchestra.run.smithi120.stderr: 90.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-21T01:19:21.924 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log 2024-06-21T01:19:21.938 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/tcmu-runner.log: 89.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/tcmu-runner.log.gz 2024-06-21T01:19:21.948 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi027.dsjkpz.log 2024-06-21T01:19:21.971 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log: /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi027.dsjkpz.log: 75.8% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-client.rgw.foo.smithi027.dsjkpz.log.gz 2024-06-21T01:19:21.977 INFO:teuthology.orchestra.run.smithi027.stderr: 91.0% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-volume.log.gz 2024-06-21T01:19:22.041 INFO:teuthology.orchestra.run.smithi120.stderr: 89.7% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.x.log.gz 2024-06-21T01:19:22.404 INFO:teuthology.orchestra.run.smithi027.stderr: 89.2% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mgr.y.log.gz 2024-06-21T01:19:22.471 INFO:teuthology.orchestra.run.smithi120.stderr: 92.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.b.log.gz 2024-06-21T01:19:22.677 INFO:teuthology.orchestra.run.smithi027.stderr: 92.4% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.c.log.gz 2024-06-21T01:19:23.376 INFO:teuthology.orchestra.run.smithi120.stderr: 93.2% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.6.log.gz 2024-06-21T01:19:23.729 INFO:teuthology.orchestra.run.smithi027.stderr: 93.3% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.2.log.gz 2024-06-21T01:19:23.729 INFO:teuthology.orchestra.run.smithi120.stderr: 93.8% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.7.log.gz 2024-06-21T01:19:23.944 INFO:teuthology.orchestra.run.smithi027.stderr: 91.1% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-mon.a.log.gz 2024-06-21T01:19:23.981 INFO:teuthology.orchestra.run.smithi027.stderr: 93.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.1.log.gz 2024-06-21T01:19:24.052 INFO:teuthology.orchestra.run.smithi120.stderr: 93.4% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.5.log.gz 2024-06-21T01:19:24.166 INFO:teuthology.orchestra.run.smithi120.stderr: 93.6% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.4.log.gz 2024-06-21T01:19:24.167 INFO:teuthology.orchestra.run.smithi027.stderr: 93.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.0.log.gz 2024-06-21T01:19:24.168 INFO:teuthology.orchestra.run.smithi120.stderr: 2024-06-21T01:19:24.169 INFO:teuthology.orchestra.run.smithi120.stderr:real 0m2.290s 2024-06-21T01:19:24.169 INFO:teuthology.orchestra.run.smithi120.stderr:user 0m8.229s 2024-06-21T01:19:24.169 INFO:teuthology.orchestra.run.smithi120.stderr:sys 0m0.323s 2024-06-21T01:19:24.660 INFO:teuthology.orchestra.run.smithi027.stderr: 93.5% -- replaced with /var/log/ceph/47651940-2f66-11ef-bc9d-c7b262605968/ceph-osd.3.log.gz 2024-06-21T01:19:24.662 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-21T01:19:24.662 INFO:teuthology.orchestra.run.smithi027.stderr:real 0m2.786s 2024-06-21T01:19:24.662 INFO:teuthology.orchestra.run.smithi027.stderr:user 0m11.889s 2024-06-21T01:19:24.662 INFO:teuthology.orchestra.run.smithi027.stderr:sys 0m0.433s 2024-06-21T01:19:24.663 INFO:tasks.cephadm:Archiving logs... 2024-06-21T01:19:24.663 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/log/ceph to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi027/log 2024-06-21T01:19:24.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-21T01:19:26.207 DEBUG:teuthology.misc:Transferring archived files from smithi120:/var/log/ceph to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi120/log 2024-06-21T01:19:26.208 DEBUG:teuthology.orchestra.run.smithi120:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-21T01:19:27.431 INFO:tasks.cephadm:Removing cluster... 2024-06-21T01:19:27.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47651940-2f66-11ef-bc9d-c7b262605968 --force 2024-06-21T01:19:27.740 DEBUG:teuthology.orchestra.run.smithi120:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47651940-2f66-11ef-bc9d-c7b262605968 --force 2024-06-21T01:19:28.030 INFO:tasks.cephadm:Removing cephadm ... 2024-06-21T01:19:28.030 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-21T01:19:28.046 DEBUG:teuthology.orchestra.run.smithi120:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-21T01:19:28.061 INFO:tasks.cephadm:Teardown complete 2024-06-21T01:19:28.062 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-06-21T01:19:28.073 INFO:teuthology.task.clock:Checking final clock skew... 2024-06-21T01:19:28.073 DEBUG:teuthology.orchestra.run.smithi027:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-21T01:19:28.089 DEBUG:teuthology.orchestra.run.smithi120:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-21T01:19:28.101 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-06-21T01:19:28.105 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-21T01:19:28.106 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-06-21T01:19:28.106 INFO:teuthology.orchestra.run.smithi027.stdout:^* hv01.front.sepia.ceph.com 3 6 377 61 +639us[ +643us] +/- 68ms 2024-06-21T01:19:28.106 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 58 -496us[ -496us] +/- 70ms 2024-06-21T01:19:28.106 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 -175us[ -175us] +/- 74ms 2024-06-21T01:19:28.106 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T01:19:28.115 INFO:teuthology.orchestra.run.smithi120.stderr:bash: line 1: ntpq: command not found 2024-06-21T01:19:28.120 INFO:teuthology.orchestra.run.smithi120.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-21T01:19:28.120 INFO:teuthology.orchestra.run.smithi120.stdout:=============================================================================== 2024-06-21T01:19:28.120 INFO:teuthology.orchestra.run.smithi120.stdout:^* hv01.front.sepia.ceph.com 3 8 377 60 +577us[ +566us] +/- 68ms 2024-06-21T01:19:28.121 INFO:teuthology.orchestra.run.smithi120.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 125 -563us[ -573us] +/- 69ms 2024-06-21T01:19:28.121 INFO:teuthology.orchestra.run.smithi120.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 61 -266us[ -277us] +/- 74ms 2024-06-21T01:19:28.121 INFO:teuthology.orchestra.run.smithi120.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-21T01:19:28.121 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-06-21T01:19:28.131 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-06-21T01:19:28.131 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-06-21T01:19:28.142 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-21T01:19:28.238 DEBUG:teuthology.orchestra.run.smithi120:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-21T01:19:28.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-21T01:19:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T01:19:28.343 DEBUG:teuthology.orchestra.run.smithi120:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-06-21T01:19:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T01:19:28.371 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-06-21T01:19:28.381 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-06-21T01:19:28.391 INFO:teuthology.task.internal:Duration was 2858.475969 seconds 2024-06-21T01:19:28.392 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-06-21T01:19:28.403 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-06-21T01:19:28.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-21T01:19:28.406 DEBUG:teuthology.orchestra.run.smithi120:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-21T01:19:28.443 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-21T01:19:28.450 INFO:teuthology.orchestra.run.smithi120.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-21T01:19:28.762 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-06-21T01:19:28.762 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2024-06-21T01:19:28.762 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-21T01:19:28.825 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi120.front.sepia.ceph.com 2024-06-21T01:19:28.826 DEBUG:teuthology.orchestra.run.smithi120:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-21T01:19:28.856 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-06-21T01:19:28.856 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-21T01:19:28.868 DEBUG:teuthology.orchestra.run.smithi120:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-21T01:19:28.938 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-06-21T01:19:28.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-21T01:19:28.941 DEBUG:teuthology.orchestra.run.smithi120:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-21T01:19:29.667 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-06-21T01:19:29.679 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-06-21T01:19:29.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-21T01:19:29.705 DEBUG:teuthology.orchestra.run.smithi120:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-21T01:19:29.730 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-06-21T01:19:29.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-21T01:19:29.748 DEBUG:teuthology.orchestra.run.smithi120:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-21T01:19:29.770 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2024-06-21T01:19:29.793 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern = core 2024-06-21T01:19:29.828 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-21T01:19:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T01:19:29.848 DEBUG:teuthology.orchestra.run.smithi120:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-21T01:19:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-21T01:19:29.882 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-06-21T01:19:29.899 INFO:teuthology.task.internal:Transferring archived files... 2024-06-21T01:19:29.901 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi027 2024-06-21T01:19:29.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-21T01:19:29.961 DEBUG:teuthology.misc:Transferring archived files from smithi120:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/lflores-2024-06-20_17:49:38-orch-squid-release-distro-default-smithi/7765834/remote/smithi120 2024-06-21T01:19:29.962 DEBUG:teuthology.orchestra.run.smithi120:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-21T01:19:30.008 INFO:teuthology.task.internal:Removing archive directory... 2024-06-21T01:19:30.008 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-21T01:19:30.010 DEBUG:teuthology.orchestra.run.smithi120:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-21T01:19:30.056 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-06-21T01:19:30.070 INFO:teuthology.task.internal:Not uploading archives. 2024-06-21T01:19:30.070 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-06-21T01:19:30.085 INFO:teuthology.task.internal:Tidying up after the test... 2024-06-21T01:19:30.085 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-21T01:19:30.087 DEBUG:teuthology.orchestra.run.smithi120:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-21T01:19:30.100 INFO:teuthology.orchestra.run.smithi027.stdout: 265313 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 21 01:19 /home/ubuntu/cephtest 2024-06-21T01:19:30.111 INFO:teuthology.orchestra.run.smithi120.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 21 01:19 /home/ubuntu/cephtest 2024-06-21T01:19:30.112 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-06-21T01:19:30.125 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-06-21T01:19:30.158 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2858.4759690761566 failure_reason: '"2024-06-21T00:54:12.634708+0000 mon.a (mon.0) 928 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_lflores@folio02 success: false 2024-06-21T01:19:30.159 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-21T01:19:30.244 INFO:teuthology.run:FAIL